2024-09-02T085431.726Z time="2024-09-02T085431Z" level=info msg="Config loaded" func=cmd/lakefs/cmd.initConfig file="cmd/root.go:151" fields.file=/home/lakefs/.lakefs.yaml file="cmd/root.go:151" phase=startup 2024-09-02T085431.726Z time="2024-09-02T085431Z" level=info msg=Config func=cmd/lakefs/cmd.initConfig file="cmd/root.go:159" actions.enabled=true actions.env.enabled=true actions.env.prefix=LAKEFSACTION_ actions.lua.net_http_enabled=false auth.api.endpoint="" auth.api.health_check_timeout=20s auth.api.skip_health_check=false auth.api.supports_invites=false auth.api.token=------ auth.authentication_api.endpoint="" auth.authentication_api.external_principals_enabled=false auth.cache.enabled=true auth.cache.jitter=3s auth.cache.size=1024 auth.cache.ttl=20s auth.cookie_auth_verification.auth_source="" auth.cookie_auth_verification.default_initial_groups="[]" auth.cookie_auth_verification.external_user_id_claim_name="" auth.cookie_auth_verification.friendly_name_claim_name="" auth.cookie_auth_verification.initial_groups_claim_name="" auth.cookie_auth_verification.persist_friendly_name=false auth.cookie_auth_verification.validate_id_token_claims="map[]" auth.encrypt.secret_key="******" auth.login_duration=168h0m0s auth.login_max_duration=336h0m0s auth.logout_redirect_url=/auth/login auth.oidc.default_initial_groups="[]" auth.oidc.friendly_name_claim_name="" auth.oidc.initial_groups_claim_name="" auth.oidc.persist_friendly_name=false auth.oidc.validate_id_token_claims="map[]" auth.remote_authenticator.default_user_group=Viewers auth.remote_authenticator.enabled=false auth.remote_authenticator.endpoint="" auth.remote_authenticator.request_timeout=10s auth.ui_config.login_cookie_names="[internal_auth_session]" auth.ui_config.login_failed_message="The credentials don't match." auth.ui_config.login_url="" auth.ui_config.logout_url="" auth.ui_config.rbac=simplified blockstore.azure.auth_method="" blockstore.azure.china_cloud=false blockstore.azure.disable_pre_signed=false blockstore.azure.disable_pre_signed_ui=true blockstore.azure.domain="" blockstore.azure.pre_signed_expiry=15m0s blockstore.azure.storage_access_key="" blockstore.azure.storage_account="" blockstore.azure.test_endpoint_url="" blockstore.azure.try_timeout=10m0s blockstore.gs.credentials_file="" blockstore.gs.credentials_json="" blockstore.gs.disable_pre_signed=false blockstore.gs.disable_pre_signed_ui=true blockstore.gs.pre_signed_expiry=15m0s blockstore.gs.s3_endpoint="https://storage.googleapis.com" blockstore.gs.server_side_encryption_customer_supplied="" blockstore.gs.server_side_encryption_kms_key_id="" blockstore.local.allowed_external_prefixes="[]" blockstore.local.import_enabled=false blockstore.local.import_hidden=false blockstore.local.path="~/lakefs/data/block" blockstore.s3.client_log_request=false blockstore.s3.client_log_retries=false blockstore.s3.credentials_file="" blockstore.s3.disable_pre_signed=false blockstore.s3.disable_pre_signed_multipart=false blockstore.s3.disable_pre_signed_ui=true blockstore.s3.discover_bucket_region=true blockstore.s3.endpoint="https://s3.eu-central-1.amazonaws.com" blockstore.s3.force_path_style=false blockstore.s3.max_retries=5 blockstore.s3.pre_signed_expiry=15m0s blockstore.s3.profile="" blockstore.s3.region=eu-central-1 blockstore.s3.server_side_encryption="" blockstore.s3.server_side_encryption_kms_key_id="" blockstore.s3.skip_verify_certificate_test_only=false blockstore.s3.web_identity.session_duration=0s blockstore.s3.web_identity.session_expiry_window=5m0s blockstore.signing.secret_key="******" blockstore.type=s3 committed.block_storage_prefix=_lakefs committed.local_cache.dir="~/lakefs/data/cache" committed.local_cache.max_uploaders_per_writer=10 committed.local_cache.metarange_proportion=0.1 committed.local_cache.range_proportion=0.9 committed.local_cache.size_bytes=1073741824 committed.permanent.max_range_size_bytes=20971520 committed.permanent.min_range_size_bytes=0 committed.permanent.range_raggedness_entries=50000 committed.sstable.memory.cache_size_bytes=400000000 database.drop_tables=false database.dynamodb.aws_access_key_id=------ database.dynamodb.aws_profile="" database.dynamodb.aws_region="" database.dynamodb.aws_secret_access_key=------ database.dynamodb.endpoint="" database.dynamodb.health_check_interval=0s database.dynamodb.max_attempts=10 database.dynamodb.max_connections=0 database.dynamodb.scan_limit=1024 database.dynamodb.table_name=specto-ai-lakefs-kvstore database.local.enable_logging=false database.local.path="~/lakefs/metadata" database.local.prefetch_size=256 database.local.sync_writes=true database.postgres.connection_max_lifetime=5m0s database.postgres.connection_string=------ database.postgres.max_idle_connections=25 database.postgres.max_open_connections=25 database.postgres.metrics=false database.postgres.scan_page_size=0 database.type=dynamodb email_subscription.enabled=false fields.file=/home/lakefs/.lakefs.yaml file="cmd/root.go:159" gateways.s3.domain_name="[s3.local.lakefs.io]" gateways.s3.fallback_url="" gateways.s3.region=eu-central-1 gateways.s3.verify_unsupported=true graveler.background.rate_limit=0 graveler.batch_dbio_transaction_markers=false graveler.commit_cache.expiry=10m0s graveler.commit_cache.jitter=2s graveler.commit_cache.size=50000 graveler.compaction_sensor_threshold=0 graveler.ensure_readable_root_namespace=true graveler.max_batch_delay=3ms graveler.repository_cache.expiry=5s graveler.repository_cache.jitter=2s graveler.repository_cache.size=1000 installation.access_key_id=------ installation.allow_inter_region_storage=true installation.fixed_id="" installation.secret_access_key=------ installation.user_name="" listen_address=":8000" logging.audit_log_level=NONE logging.file_max_size_mb=102400 logging.files_keep=100 logging.format=text logging.level=INFO logging.output="[-]" logging.trace_request_headers=false phase=startup security.audit_check_interval=24h0m0s security.audit_check_url="https://audit.lakefs.io/audit" security.check_latest_version=true security.check_latest_version_cache=1h0m0s stats.address="https://stats.lakefs.io" stats.enabled=false stats.extended=false stats.flush_interval=30s stats.flush_size=100 tls.cert_file="" tls.enabled=false tls.key_file="" ugc.prepare_interval=1m0s ugc.prepare_max_file_size=20971520 ui.enabled=true ui.snippets="[]" usage_report.enabled=false usage_report.flush_interval=5m0s 2024-09-02T085431.728Z time="2024-09-02T085431Z" level=info msg="lakeFS run" func=cmd/lakefs/cmd.glob..func9 file="cmd/run.go:123" version=1.28.1 2024-09-02T085431.778Z time="2024-09-02T085431Z" level=info msg="KV valid" func=pkg/kv.ValidateSchemaVersion file="build/pkg/kv/migration.go:68" version=4 2024-09-02T085431.778Z time="2024-09-02T085431Z" level=info msg="initialized Auth service" func=pkg/auth.NewAuthService file="build/pkg/auth/service.go:217" service=auth_service 2024-09-02T085449.039Z time="2024-09-02T085449Z" level=warning msg="Tried to to get AWS account ID for BI" func="pkg/cloud/aws.(*MetadataProvider).GetMetadata.func1" file="build/pkg/cloud/aws/metadata.go:81" error="operation error STS: GetCallerIdentity, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , request send failed, Post \"https://sts.eu-central-1.amazonaws.com/\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)" 2024-09-02T085449.039Z time="2024-09-02T085449Z" level=warning msg="Failed to to get AWS account ID for BI" func="pkg/cloud/aws.(*MetadataProvider).GetMetadata" file="build/pkg/cloud/aws/metadata.go:87" error="operation error STS: GetCallerIdentity, exceeded maximum number of attempts, 3, https response error StatusCode: 0, RequestID: , request send failed, Post \"https://sts.eu-central-1.amazonaws.com/\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)" 2024-09-02T085449.042Z time="2024-09-02T085449Z" level=info msg="initialize blockstore adapter" func=pkg/block/factory.buildBlockAdapter file="build/pkg/block/factory/build.go:41" type=s3 2024-09-02T085449.042Z time="2024-09-02T085449Z" level=info msg="initialized blockstore adapter" func=pkg/block/factory.buildS3Adapter file="build/pkg/block/factory/build.go:121" type=s3 2024-09-02T085449.042Z time="2024-09-02T085449Z" level=info msg="initialize blockstore adapter" func=pkg/block/factory.buildBlockAdapter file="build/pkg/block/factory/build.go:41" type=s3 2024-09-02T085449.042Z time="2024-09-02T085449Z" level=info msg="initialized blockstore adapter" func=pkg/block/factory.buildS3Adapter file="build/pkg/block/factory/build.go:121" type=s3 2024-09-02T085449.083Z time="2024-09-02T085449Z" level=info msg="initialize OpenAPI server" func=pkg/api.Serve file="build/pkg/api/serve.go:37" service=api_gateway 2024-09-02T085449.256Z time="2024-09-02T085449Z" level=info msg="initialized S3 Gateway handler" func=pkg/gateway.NewHandler file="build/pkg/gateway/handler.go:128" s3_bare_domain="[s3.local.lakefs.io]" s3_region=eu-central-1 2024-09-02T085449.256Z time="2024-09-02T085449Z" level=info msg="starting HTTP server" func=cmd/lakefs/cmd.glob..func9 file="cmd/run.go:329" listen_address=":8000" 2024-09-02T085449.256Z lakeFS 1.28.1 - Up and running (^C to shutdown)... 2024-09-02T085449.256Z ██╗ █████╗ ██╗ ██╗███████╗███████╗███████╗ 2024-09-02T085449.256Z ██║ ██╔══██╗██║ ██╔╝██╔════╝██╔════╝██╔════╝ 2024-09-02T085449.256Z ██║ ███████║█████╔╝ █████╗ █████╗ ███████╗ 2024-09-02T085449.256Z ██║ ██╔══██║██╔═██╗ ██╔══╝ ██╔══╝ ╚════██║ 2024-09-02T085449.256Z ███████╗██║ ██║██║ ██╗███████╗██║ ███████║ 2024-09-02T085449.256Z ╚══════╝╚═╝ ╚═╝╚═╝ ╚═╝╚══════╝╚═╝ ╚══════╝ 2024-09-02T085449.256Z │ 2024-09-02T085449.256Z │ If you're running lakeFS locally for the first time, 2024-09-02T085449.256Z │ complete the setup process at http://127.0.0.1:8000/setup 2024-09-02T085449.256Z │ 2024-09-02T085449.256Z │ 2024-09-02T085449.256Z │ For more information on how to use lakeFS, 2024-09-02T085449.256Z │ check out the docs at https://docs.lakefs.io/quickstart/ 2024-09-02T085449.256Z │ 2024-09-02T085449.256Z │ 2024-09-02T085449.256Z │ For support or any other question, >(.＿.)< 2024-09-02T085449.256Z │ join our Slack channel https://docs.lakefs.io/slack ( )_ 2024-09-02T085449.256Z │ 2024-09-02T085449.256Z Version 1.28.1 2024-09-02T085519.052Z time="2024-09-02T085519Z" level=error msg="Audit check failed" func="pkg/version.(*AuditChecker).CheckAndLog" file="build/pkg/version/audit.go:113" check_url="https://audit.lakefs.io/audit" error="Get \

what is: security •

security.audit_check_interval

(duration : 24h)

- Duration in which we check for security audit. can I disable this? or set it to infinity?

and as it is recommended in the documentation, the S3 access policy is:

resource "aws_iam_policy" "lakefs_s3_policy" {
  name        = "lakefs-s3-policy"
  description = "Policy for LakeFS to access S3"
  policy      = jsonencode({
    Version = "2012-10-17",
    Statement = [
      {
        Sid = "lakeFSObjects",
        Action = [
          "s3:GetObject",
          "s3:PutObject",
          "s3:AbortMultipartUpload",
          "s3:ListMultipartUploadParts"
        ],
        Effect = "Allow",
        Resource = "${aws_s3_bucket.lakefs_bucket.arn}/*"
      },
      {
        Sid = "lakeFSBucket",
        Action = [
          "s3:ListBucket",
          "s3:GetBucketLocation",
          "s3:ListBucketMultipartUploads"
        ],
        Effect = "Allow",
        Resource = aws_s3_bucket.lakefs_bucket.arn
      }
    ]
  })
}

I've also tried to grant all access to s3. but seems not to be the issue

ah, and by the way: <html> <head><title>403 Forbidden</title></head> <body> <center><h1>403 Forbidden</h1></center> </body> </html> <!-- a padding to disable MSIE and Chrome friendly error page --> <!-- a padding to disable MSIE and Chrome friendly error page --> <!-- a padding to disable MSIE and Chrome friendly error page --> <!-- a padding to disable MSIE and Chrome friendly error page --> <!-- a padding to disable MSIE and Chrome friendly error page --> <!-- a padding to disable MSIE and Chrome friendly error page --> this is the error message in the web UI

Also, when you claim you can write data to s3, it is through lakeFS?

I do can create repositories with the example data contained

image.png

In order to disable the audit messages, please set security.check_latest_version as false (or use SECURITY_CHECK_LATEST_VERSION env var)

However, by setting this, you wont get any alerts on new lakeFS versions or security updates

still got the audit log error:

2024-09-02T101645.745Z time="2024-09-02T101645Z" level=error msg="Audit check failed" func="pkg/version.(*AuditChecker).CheckAndLog" file="build/pkg/version/audit.go:113" check_url="https://audit.lakefs.io/audit" error="Get \"https://audit.lakefs.io/audit?installation_id=xxxxxxxxx&version=1.28.1\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" version=1.28.1

so, seems to be an issue of the web ui?

Can you please generate a HAR file from your browser while trying to upload and getting the 403?

Also, please try a different browser

what I see in the logs as well is: time="2024-09-02T133101Z" level=debug msg="failed to check latest version in releases" func="pkg/api.(*Controller).getVersionConfig" file="build/pkg/api/controller.go:5073" error="Get \"https://api.github.com/repos/treeverse/lakeFS/releases/latest\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" service=api_gateway time="2024-09-02T132915Z" level=error msg="Audit check failed" func="pkg/version.(*AuditChecker).CheckAndLog" file="build/pkg/version/audit.go:113" check_url="https://audit.lakefs.io/audit" error="Get \"https://audit.lakefs.io/audit?installation_id=6a6e55e5-c2a3-4d45-8d89-eb3e43cb4944&amp;version=1.25.0\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)" version=1.25.0 time="2024-09-02T133421Z" level=error msg="Failed to get region for bucket, falling back to default region" func="pkg/block/s3.(*ClientCache).refreshBucketRegion" file="build/pkg/block/s3/client_cache.go:151" default_region=eu-central-1 error="bucket not found" host=lakefs.spectomedical.com method=GET operation_id=ListObjects path="/api/v1/repositories/test-repository2/refs/main/objects/ls?prefix=&amount=100&after=&delimiter=%2F&presign=false" request_id=eb475810-9b06-4a96-92b7-a0b37b92b830 service_name=rest_api user=admin the first two might not related to my problem, but still, I'd be happy to resolve them. the last one I think is the relevant one. I will double check, that the s3 is reachable from the lakefs server.

I've again created a test repo. And I see on AWS, that the repo has been created: I can also look at a random image item in the test repo data: but accessing the README.md of the test data repo there is an error:

my current config:

{ name  = "LAKEFS_DATABASE_TYPE", value = "dynamodb" },  # Specify the internal database to be DynamoDB
{ name  = "LAKEFS_DATABASE_DYNAMODB_TABLE_NAME", value="${var.project_name}-${var.lakefs_dynamodb_table_name}" },
{ name  = "LAKEFS_BLOCKSTORE_TYPE", value = "s3" },  # Specify the internal storage type to be S3
{ name  = "LAKEFS_BLOCKSTORE_S3_REGION", value = var.aws_region },
{ name  = "LAKEFS_BLOCKSTORE_S3_ENDPOINT", value = "<https://s3>.${var.aws_region}.<http://amazonaws.com/|amazonaws.com/>" },
{ name  = "LAKEFS_BLOCKSTORE_S3_FORCE_PATH_STYLE", value = "true"},
{ name  = "LAKEFS_BLOCKSTORE_s3_DISCOVER_BUCKET_REGION", value = "false"},
{ name  = "AWS_REGION", value = var.aws_region },
{ name  = "LAKEFS_GATEWAYS_S3_REGION", value = var.aws_region },
{ name  = "LAKEFS_AUTH_ENCRYPT_SECRET_KEY", value = random_password.lakefs_encryption_secret.result },
{ name  = "LAKEFS_LISTEN_ADDRESS", value = ":8000" },
{ name  = "LAKEFS_STATS_ENABLED", value = "false" },  # Disable sending statistics to treeverse
{ name = "LAKEFS_LOGGING_LEVEL", value = "TRACE"},
{ name  = "LAKEFS_EMAIL_SUBSCRIPTION_ENABLED", value = "false" }

interestingly, after re-deployment, the upload over lakectl does not work anymore: lakectl fs upload --recursive --source /mnt/c/Users/ChristophJud/Data/BraTS2023/BraTS-MEN-Train/BraTS-MEN-00034-000 lakefs://test-repository4/main/test_dataset --verbose diff 'local:///mnt/c/Users/ChristophJud/Data/BraTS2023/BraTS-MEN-Train/BraTS-MEN-00034-000/' <--> 'lakefs://test-repository4/main/test_dataset'... upload BraTS-MEN-00034-000-seg.nii.gz ... done! [95.74KB in 61ms] upload BraTS-MEN-00034-000-t2f.nii.gz ... fail! [3.64MB in 188ms] upload BraTS-MEN-00034-000-t2w.nii.gz ... fail! [3.70MB in 188ms] upload BraTS-MEN-00034-000-t1c.nii.gz ... fail! [4.46MB in 188ms] upload BraTS-MEN-00034-000-t1n.nii.gz ... done! [4.67MB in 164ms] upload BraTS-MEN-00034-000-seg.nii.gz failed: link object to backing store: request failed (403 Forbidden)

accessing the README.md file on the console with: lakectl fs download lakefs://<repository>/<branch>/<file-path> <destination-path> works

ok, now I can again create a repository, I can download a file and randomly look at an image in the test repo data. but again, if I browsed the README.md which I can download with lakectl, I'm getting an error:

ah, and the upload does not work as well: lakectl fs upload --recursive --source /mnt/c/Users/ChristophJud/Data/BraTS2023/BraTS-MEN-Train/BraTS-MEN-00034-000 lakefs://test-repository5/main/test_dataset --verbose diff 'local:///mnt/c/Users/ChristophJud/Data/BraTS2023/BraTS-MEN-Train/BraTS-MEN-00034-000/' <--> 'lakefs://test-repository5/main/test_dataset'... upload BraTS-MEN-00034-000-seg.nii.gz ... done! [95.74KB in 66ms] upload BraTS-MEN-00034-000-t1c.nii.gz ... done! [4.67MB in 168ms] upload BraTS-MEN-00034-000-t2w.nii.gz ... done! [4.77MB in 168ms] upload BraTS-MEN-00034-000-t1n.nii.gz ... fail! [819.20KB in 209ms] upload BraTS-MEN-00034-000-t2f.nii.gz ... fail! [3.38MB in 209ms] upload BraTS-MEN-00034-000-seg.nii.gz failed: link object to backing store: request failed (403 Forbidden) Error executing command.

I wonder if it was a permission issue. however, as creation of the repo works and downloading (with lakectl) as well I don't see where the problem comes from

I forgot that it is enabled by default

but still, the README.md does not show up

this works: but the README.md not. I tried it with Brave and Edge.

this works: lakectl fs download lakefs://test-repository5/main/README.md /tmp download: lakefs://test-repository5/main/README.md to /tmp/README.md

I figured it out!!

I disabled the whole firewall

I mean the application firewall

I have to investigate

it is the: LFI_QUERYSTRING rule

thank's anyways for your help!