HT
08/24/2023, 11:57 PMlakefs_client
python sdk:
File "/path/lakefs_helper.py", line 102, in merge
return self.lakefs.refs.merge_into_branch(
File "/path/venv/lib64/python3.10/site-packages/lakefs_client/api/refs_api.py", line 869, in merge_into_branch
return self.merge_into_branch_endpoint.call_with_http_info(**kwargs)
File "/path/venv/lib64/python3.10/site-packages/lakefs_client/api_client.py", line 835, in call_with_http_info
return self.api_client.call_api(
File "/path/venv/lib64/python3.10/site-packages/lakefs_client/api_client.py", line 409, in call_api
return self.__call_api(resource_path, method,
File "/path/venv/lib64/python3.10/site-packages/lakefs_client/api_client.py", line 203, in __call_api
raise e
File "/path/venv/lib64/python3.10/site-packages/lakefs_client/api_client.py", line 196, in __call_api
response_data = self.request(
File "/path/venv/lib64/python3.10/site-packages/lakefs_client/api_client.py", line 455, in request
return <http://self.rest_client.POST|self.rest_client.POST>(url,
File "/path/venv/lib64/python3.10/site-packages/lakefs_client/rest.py", line 267, in POST
return self.request("POST", url,
File "/path/venv/lib64/python3.10/site-packages/lakefs_client/rest.py", line 224, in request
raise ServiceException(http_resp=r)
lakefs_client.exceptions.ServiceException: (503)
Reason: Service Unavailable
HTTP response headers: HTTPHeaderDict({'content-length': '118', 'content-type': 'text/plain', 'date': 'Thu, 24 Aug 2023 23:47:57 GMT'})
HTTP response body: upstream connect error or disconnect/reset before headers. retried and the latest reset reason: connection termination
My helper function that do the merge is:
def merge(self, source_branch, dest_branch, metadata={}, message=None):
try:
# Check if there is a need to merge:
diffs = self.diff(left_ref=dest_branch, right_ref=source_branch)
if len(diffs) == 0:
print(f"No changes detected. Skipping merge {source_branch} to {dest_branch}")
return
if message:
merge = models.Merge(message=message, metadata=metadata)
else:
merge = models.Merge(metadata=metadata)
return self.lakefs.refs.merge_into_branch(
repository=self.repo_name,
source_ref=source_branch,
destination_branch=dest_branch,
merge=merge,
)
except Exception as e:
# traceback.print_exc()
print(f"ERROR: Failed to merge {source_branch} to {dest_branch}")
raise e
With self.lakefs as :
configuration = lakefs_client.Configuration()
configuration.username = conf["access_key_id"]
configuration.password = conf["secret_access_key"]
configuration.host = conf["endpoint"]
self.lakefs = LakeFSClient(configuration)
But then if I do the merge of that 2 exact same branch via the UI, it succeed !
Our self deployed server running : 107.0
And the lakefs_client is 0.107.0Lynn Rozen
08/25/2023, 7:46 AMAriel Shaqed (Scolnicov)
08/27/2023, 9:20 AMReason: Service Unavailable
HTTP response headers: HTTPHeaderDict({'content-length': '118', 'content-type': 'text/plain', 'date': 'Thu, 24 Aug 2023 23:47:57 GMT'})
HTTP response body: upstream connect error or disconnect/reset before headers. retried and the latest reset reason: connection termination
indicate either an error on a "gateway" (could be a proxy, reverse proxy, load-balancer, K8s ingress controller, ...) or possibly in lakeFS itself. But they hide the underlying cause. Do you have logs from the lakeFS server itself at the time of failure? If so, I'd be grateful if you could scrub them of sensitive data and send them.
Thanks!HT
08/28/2023, 4:21 AMfrom datetime import datetime
import lakefs_client
from lakefs_client import models
from lakefs_client.client import LakeFSClient
from livsdk.smartfs import load_rclone_profile
conf = load_rclone_profile("locallakefs")
branch_name = f"test_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
repo='test-repo'
# lakeFS credentials and endpoint
configuration = lakefs_client.Configuration()
configuration.username = conf["access_key_id"]
configuration.password = conf["secret_access_key"]
configuration.host = conf["endpoint"]
client = LakeFSClient(configuration)
branch_name = f"test-merge_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
commit_origin=client.branches_api.create_branch(repository=repo, branch_creation=models.BranchCreation(name=branch_name, source='main'))
with open('/dev/shm/test.csv', 'rb') as f:
client.objects_api.upload_object(repository=repo, branch=branch_name, path='file.csv', content=f)
client.commits_api.commit(
repository=repo,
branch=branch_name,
commit_creation=models.CommitCreation(message='Added a CSV file!', metadata={'using': 'python_api'}))
# Create a second branch
branch_second = branch_name + "_bis"
client.branches_api.create_branch(repository=repo, branch_creation=models.BranchCreation(name=branch_second, source=commit_origin))
with open('/dev/shm/new.csv', 'rb') as f:
client.objects_api.upload_object(repository=repo, branch=branch_second, path='new.csv', content=f)
client.commits_api.commit(
repository=repo,
branch=branch_second,
commit_creation=models.CommitCreation(message='Added new file!', metadata={'using': 'python_api'}))
# merge
merge = models.Merge(metadata={})
client.refs.merge_into_branch(repository=repo,
source_ref=branch_name, destination_branch=branch_second,
merge=merge
)
That crash with:
Traceback (most recent call last):
File "/data/hieu/opt/python-venv/fastai/lib64/python3.10/site-packages/urllib3/connectionpool.py", line 790, in urlopen
response = self._make_request(
File "/data/hieu/opt/python-venv/fastai/lib64/python3.10/site-packages/urllib3/connectionpool.py", line 536, in _make_request
response = conn.getresponse()
File "/data/hieu/opt/python-venv/fastai/lib64/python3.10/site-packages/urllib3/connection.py", line 461, in getresponse
httplib_response = super().getresponse()
File "/usr/lib64/python3.10/http/client.py", line 1374, in getresponse
response.begin()
File "/usr/lib64/python3.10/http/client.py", line 318, in begin
version, status, reason = self._read_status()
File "/usr/lib64/python3.10/http/client.py", line 287, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
And the lakefs docker log :
lakeFS 0.107.1 - Up and running (^C to shutdown)...
██╗ █████╗ ██╗ ██╗███████╗███████╗███████╗
██║ ██╔══██╗██║ ██╔╝██╔════╝██╔════╝██╔════╝
██║ ███████║█████╔╝ █████╗ █████╗ ███████╗
██║ ██╔══██║██╔═██╗ ██╔══╝ ██╔══╝ ╚════██║
███████╗██║ ██║██║ ██╗███████╗██║ ███████║
╚══════╝╚═╝ ╚═╝╚═╝ ╚═╝╚══════╝╚═╝ ╚══════╝
│
│ lakeFS running in quickstart mode.
│ Login at <http://127.0.0.1:8000/>
│
│ Access Key ID : AKIAIOSFOLQUICKSTART
│ Secret Access Key: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
│
│
│ For more information on how to use lakeFS,
│ check out the docs at <https://docs.lakefs.io/quickstart/>
│
│
│ For support or any other question, >(._.)<
│ join our Slack channel <https://docs.lakefs.io/slack> ( )_
│
Version 0.107.1
2023/08/28 04:17:31 http: panic serving 172.17.0.1:52354: assignment to entry in nil map
goroutine 1283 [running]:
net/http.(*conn).serve.func1()
/usr/local/go/src/net/http/server.go:1854 +0xbf
panic({0x1d836e0, 0x57956d0})
/usr/local/go/src/runtime/panic.go:890 +0x263
<http://github.com/treeverse/lakefs/pkg/graveler.(*Graveler).Merge.func1(0xc00fa7ea00)|github.com/treeverse/lakefs/pkg/graveler.(*Graveler).Merge.func1(0xc00fa7ea00)>
/build/pkg/graveler/graveler.go:2557 +0xa3e
<http://github.com/treeverse/lakefs/pkg/graveler/ref.(*Manager).BranchUpdate|github.com/treeverse/lakefs/pkg/graveler/ref.(*Manager).BranchUpdate>(0xc00614e240, {0x57b0d80, 0xc00c97a3f0}, 0xc000012438, {0xc00e11104a, 0x1e}, 0xc00e9843c0)
/build/pkg/graveler/ref/manager.go:403 +0x8c
<http://github.com/treeverse/lakefs/pkg/graveler.(*Graveler).retryBranchUpdate.func1()|github.com/treeverse/lakefs/pkg/graveler.(*Graveler).retryBranchUpdate.func1()>
/build/pkg/graveler/graveler.go:2028 +0x79
<http://github.com/cenkalti/backoff/v4.RetryNotifyWithTimer.func1()|github.com/cenkalti/backoff/v4.RetryNotifyWithTimer.func1()>
/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.2.1/retry.go:18 +0x1b
<http://github.com/cenkalti/backoff/v4.doRetryNotify[...]|github.com/cenkalti/backoff/v4.doRetryNotify[...]>(0xc0009ae680?, {0x57a5758, 0xc00614f440}, 0x0, {0x0, 0x0?})
/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.2.1/retry.go:88 +0x152
<http://github.com/cenkalti/backoff/v4.RetryNotifyWithTimer(0x7f98ae9e03f8|github.com/cenkalti/backoff/v4.RetryNotifyWithTimer(0x7f98ae9e03f8>?, {0x57a5758?, 0xc00614f440?}, 0x4232eb?, {0x0?, 0x0?})
/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.2.1/retry.go:61 +0x65
<http://github.com/cenkalti/backoff/v4.RetryNotify(...)|github.com/cenkalti/backoff/v4.RetryNotify(...)>
/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.2.1/retry.go:49
<http://github.com/cenkalti/backoff/v4.Retry(...)|github.com/cenkalti/backoff/v4.Retry(...)>
/go/pkg/mod/github.com/cenkalti/backoff/v4@v4.2.1/retry.go:38
<http://github.com/treeverse/lakefs/pkg/graveler.(*Graveler).retryBranchUpdate|github.com/treeverse/lakefs/pkg/graveler.(*Graveler).retryBranchUpdate>(0xc0001321c0, {0x57b0d80, 0xc00c97a3f0}, 0xc000012438, {0xc00e11104a, 0x1e}, 0xc00e9843c0, {0x210a9cd, 0x5})
/build/pkg/graveler/graveler.go:2025 +0x22a
<http://github.com/treeverse/lakefs/pkg/graveler.(*Graveler).Merge(0xc0001321c0|github.com/treeverse/lakefs/pkg/graveler.(*Graveler).Merge(0xc0001321c0>, {0x57b0d80?, 0xc00c97a3f0}, 0xc000012438, {0xc00e11104a, 0x1e}, {0xc00e111029, 0x1a}, {{0xc00ef921d0, 0xa}, ...}, ...)
/build/pkg/graveler/graveler.go:2505 +0x425
<http://github.com/treeverse/lakefs/pkg/catalog.(*Catalog).Merge(0xc006142000|github.com/treeverse/lakefs/pkg/catalog.(*Catalog).Merge(0xc006142000>, {0x57b0d80, 0xc00001dfb0}, {0xc00e11101a, 0x9}, {0xc00e11104a, 0x1e}, {0xc00e111029, 0x1a}, {0xc00ef921d0, ...}, ...)
/build/pkg/catalog/catalog.go:1721 +0x4e9
<http://github.com/treeverse/lakefs/pkg/api.(*Controller).MergeIntoBranch(0xc00ca3e500|github.com/treeverse/lakefs/pkg/api.(*Controller).MergeIntoBranch(0xc00ca3e500>, {0x57adc00, 0xc00e10c930}, 0xc00ca3e900, {0x0?, 0xc00f19c1c0?, 0x0?}, {0xc00e11101a, 0x9}, {0xc00e111029, ...}, ...)
/build/pkg/api/controller.go:3905 +0x303
<http://github.com/treeverse/lakefs/pkg/api.(*ServerInterfaceWrapper).MergeIntoBranch.func1(|github.com/treeverse/lakefs/pkg/api.(*ServerInterfaceWrapper).MergeIntoBranch.func1(>{0x57adc00?, 0xc00e10c930?}, 0x1c8b100?)
/build/pkg/api/lakefs.gen.go:24486 +0xbb
<http://github.com/treeverse/lakefs/pkg/api.(*ServerInterfaceWrapper).MergeIntoBranch(0xc00cc44f90|github.com/treeverse/lakefs/pkg/api.(*ServerInterfaceWrapper).MergeIntoBranch(0xc00cc44f90>, {0x57adc00, 0xc00e10c930}, 0xc00ca3e800)
/build/pkg/api/lakefs.gen.go:24493 +0x7bc
net/http.HandlerFunc.ServeHTTP(0xc00db14bd0?, {0x57adc00?, 0xc00e10c930?}, 0x7f995e8ce108?)
/usr/local/go/src/net/http/server.go:2122 +0x2f
<http://github.com/treeverse/lakefs/pkg/api.MetricsMiddleware.func1.1(|github.com/treeverse/lakefs/pkg/api.MetricsMiddleware.func1.1(>{0x7f9937331b98?, 0xc00001dda0}, 0x57adc00?)
/build/pkg/api/metrics_middleware.go:27 +0xfc
net/http.HandlerFunc.ServeHTTP(0x57adc30?, {0x7f9937331b98?, 0xc00001dda0?}, 0x390000c0009af301?)
/usr/local/go/src/net/http/server.go:2122 +0x2f
<http://github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1({0x57adc30|github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1({0x57adc30>?, 0xc0008197a0?}, 0xc00ca3e800)
/go/pkg/mod/github.com/prometheus/client_golang@v1.16.0/prometheus/promhttp/instrument_server.go:147 +0xc5
net/http.HandlerFunc.ServeHTTP(0x57b0d80?, {0x57adc30?, 0xc0008197a0?}, 0x5792340?)
/usr/local/go/src/net/http/server.go:2122 +0x2f
<http://github.com/treeverse/lakefs/pkg/api.AuthMiddleware.func1.1(|github.com/treeverse/lakefs/pkg/api.AuthMiddleware.func1.1(>{0x57adc30, 0xc0008197a0}, 0xc00ca3e700)
/build/pkg/api/auth_middleware.go:108 +0x4b0
net/http.HandlerFunc.ServeHTTP(0x1f4cb00?, {0x57adc30?, 0xc0008197a0?}, 0xc?)
/usr/local/go/src/net/http/server.go:2122 +0x2f
<http://github.com/treeverse/lakefs/pkg/httputil.DefaultLoggingMiddleware.func1.1(|github.com/treeverse/lakefs/pkg/httputil.DefaultLoggingMiddleware.func1.1(>{0x57af9a0?, 0xc00ea9e000}, 0xffffffffffffff01?)
/build/pkg/httputil/logging.go:87 +0x645
net/http.HandlerFunc.ServeHTTP(0xc00ca3e300?, {0x57af9a0?, 0xc00ea9e000?}, 0xc00cbf3090?)
/usr/local/go/src/net/http/server.go:2122 +0x2f
<http://github.com/treeverse/lakefs/pkg/api.OapiRequestValidatorWithOptions.func1.1(|github.com/treeverse/lakefs/pkg/api.OapiRequestValidatorWithOptions.func1.1(>{0x57af9a0, 0xc00ea9e000}, 0xc00ca3e100)
/build/pkg/api/serve.go:170 +0x2e3
net/http.HandlerFunc.ServeHTTP(0xc00dafd3e0?, {0x57af9a0?, 0xc00ea9e000?}, 0xc0005ae2e8?)
/usr/local/go/src/net/http/server.go:2122 +0x2f
<http://github.com/go-chi/chi/v5.(*ChainHandler).ServeHTTP(0x1d5abe0|github.com/go-chi/chi/v5.(*ChainHandler).ServeHTTP(0x1d5abe0>?, {0x57af9a0?, 0xc00ea9e000?}, 0xc00e111005?)
/go/pkg/mod/github.com/go-chi/chi/v5@v5.0.0/chain.go:31 +0x2c
<http://github.com/go-chi/chi/v5.(*Mux).routeHTTP(0xc0001ad380|github.com/go-chi/chi/v5.(*Mux).routeHTTP(0xc0001ad380>, {0x57af9a0, 0xc00ea9e000}, 0xc00ca3e100)
/go/pkg/mod/github.com/go-chi/chi/v5@v5.0.0/mux.go:436 +0x1f9
net/http.HandlerFunc.ServeHTTP(0x57b0cd8?, {0x57af9a0?, 0xc00ea9e000?}, 0x66db4e0?)
/usr/local/go/src/net/http/server.go:2122 +0x2f
<http://github.com/go-chi/chi/v5.(*Mux).ServeHTTP(0xc0001ad380|github.com/go-chi/chi/v5.(*Mux).ServeHTTP(0xc0001ad380>, {0x57af9a0, 0xc00ea9e000}, 0xc00ca3e000)
/go/pkg/mod/github.com/go-chi/chi/v5@v5.0.0/mux.go:87 +0x32a
<http://github.com/treeverse/lakefs/cmd/lakefs/cmd.glob..func8.3({0x57af9a0|github.com/treeverse/lakefs/cmd/lakefs/cmd.glob..func8.3({0x57af9a0>, 0xc00ea9e000}, 0xc0009afad0?)
/build/cmd/lakefs/cmd/run.go:342 +0x102
net/http.HandlerFunc.ServeHTTP(0x0?, {0x57af9a0?, 0xc00ea9e000?}, 0x8663d4?)
/usr/local/go/src/net/http/server.go:2122 +0x2f
net/http.serverHandler.ServeHTTP({0x57a8750?}, {0x57af9a0, 0xc00ea9e000}, 0xc00ca3e000)
/usr/local/go/src/net/http/server.go:2936 +0x316
net/http.(*conn).serve(0xc00eea0480, {0x57b0d80, 0xc00e386ea0})
/usr/local/go/src/net/http/server.go:1995 +0x612
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:3089 +0x5ed
Ariel Shaqed (Scolnicov)
08/28/2023, 6:33 AMHT
09/01/2023, 8:22 AM