From 2c36141f35acaf85944df5633341b74a643a4cef Mon Sep 17 00:00:00 2001 From: skyflow-bharti Date: Fri, 12 Sep 2025 15:45:03 +0530 Subject: [PATCH 01/10] SK-2292 internal release for v1 --- .github/workflows/internal_release.yml | 0 .github/workflows/shared-build-and-deploy.yml | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 .github/workflows/internal_release.yml create mode 100644 .github/workflows/shared-build-and-deploy.yml diff --git a/.github/workflows/internal_release.yml b/.github/workflows/internal_release.yml new file mode 100644 index 00000000..e69de29b diff --git a/.github/workflows/shared-build-and-deploy.yml b/.github/workflows/shared-build-and-deploy.yml new file mode 100644 index 00000000..e69de29b From f4e1cfd9952b7d0a6a8d667a7de5a43dbb365c51 Mon Sep 17 00:00:00 2001 From: skyflow-bharti Date: Fri, 12 Sep 2025 19:06:46 +0530 Subject: [PATCH 02/10] SK-2293 update workflow --- .github/workflows/internal_release.yml | 23 +++++ .github/workflows/shared-build-and-deploy.yml | 83 +++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/.github/workflows/internal_release.yml b/.github/workflows/internal_release.yml index e69de29b..912b052a 100644 --- a/.github/workflows/internal_release.yml +++ b/.github/workflows/internal_release.yml @@ -0,0 +1,23 @@ +name: Internal Release + +on: + push: + tags-ignore: + - '*.*' + paths-ignore: + - "setup.py" + - "*.yml" + - "*.md" + - "skyflow/version.py" + - "samples/**" + branches: + - release/* + +jobs: + build-and-deploy: + uses: ./.github/workflows/shared-build-and-deploy.yml + with: + ref: ${{ github.ref_name }} + tag: 'internal' + secrets: inherit + \ No newline at end of file diff --git a/.github/workflows/shared-build-and-deploy.yml b/.github/workflows/shared-build-and-deploy.yml index e69de29b..80628d64 100644 --- a/.github/workflows/shared-build-and-deploy.yml +++ b/.github/workflows/shared-build-and-deploy.yml @@ -0,0 +1,83 @@ +name: Shared Build and Deploy + +on: + workflow_call: + inputs: + ref: + description: 'Git reference to use (e.g., main or branch name)' + required: true + type: string + + tag: + description: 'Release Tag' + required: true + type: string + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel twine + + - name: Resolve Branch for the Tagged Commit + id: resolve-branch + if: ${{ inputs.tag == 'beta' || inputs.tag == 'public' }} + run: | + TAG_COMMIT=$(git rev-list -n 1 ${{ github.ref_name }}) + + BRANCH_NAME=$(git branch -r --contains $TAG_COMMIT | grep -o 'origin/.*' | sed 's|origin/||' | head -n 1) + + if [ -z "$BRANCH_NAME" ]; then + echo "Error: Could not resolve branch for the tag." + exit 1 + fi + + echo "Resolved Branch Name: $BRANCH_NAME" + echo "branch_name=$BRANCH_NAME" >> $GITHUB_ENV + + - name: Get Previous tag + id: previoustag + uses: WyriHaximus/github-action-get-previous-tag@v1 + with: + fallback: 1.0.0 + + - name: Bump Version + run: | + chmod +x ./ci-scripts/bump_version.sh + if ${{ inputs.tag == 'internal' }}; then + ./ci-scripts/bump_version.sh "${{ steps.previoustag.outputs.tag }}" "$(git rev-parse --short "$GITHUB_SHA")" + else + ./ci-scripts/bump_version.sh "${{ steps.previoustag.outputs.tag }}" + fi + + - name: Commit changes + run: | + git config user.name "${{ github.actor }}" + git config user.email "${{ github.actor }}@users.noreply.github.com" + + git add setup.py + git add skyflow/version.py + + if [[ "${{ inputs.tag }}" == "internal" ]]; then + VERSION="${{ steps.previoustag.outputs.tag }}.dev0+$(git rev-parse --short $GITHUB_SHA)" + COMMIT_MESSAGE="[AUTOMATED] Private Release $VERSION" + git commit -m "$COMMIT_MESSAGE" + git push origin ${{ github.ref_name }} -f + fi + + - name: Build and Publish to JFrog Artifactory + if: ${{ inputs.tag == 'internal' }} + env: + TWINE_USERNAME: ${{ secrets.JFROG_USERNAME }} + TWINE_PASSWORD: ${{ secrets.JFROG_PASSWORD }} + run: | + python setup.py sdist bdist_wheel + twine upload --repository-url https://prekarilabs.jfrog.io/artifactory/api/pypi/skyflow-python/ dist/* \ No newline at end of file From ba58e984052c19baa94610cdce64038b6b0b72fd Mon Sep 17 00:00:00 2001 From: skyflow-bharti Date: Fri, 12 Sep 2025 13:37:24 +0000 Subject: [PATCH 03/10] [AUTOMATED] Private Release 2.1.0b1.dev0+f4e1cfd --- setup.py | 2 +- skyflow/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index c7756728..1739e441 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ if sys.version_info < (3, 7): raise RuntimeError("skyflow requires Python 3.7+") -current_version = '1.15.1' +current_version = '2.1.0b1-dev.f4e1cfd' setup( name='skyflow', diff --git a/skyflow/version.py b/skyflow/version.py index 4d6b1a07..1ce30dae 100644 --- a/skyflow/version.py +++ b/skyflow/version.py @@ -1 +1 @@ -SDK_VERSION = '1.15.1' \ No newline at end of file +SDK_VERSION = '2.1.0b1-dev.f4e1cfd' \ No newline at end of file From 8efd1cd5ead7aadaa121eaff6bb6457d1ecad4b3 Mon Sep 17 00:00:00 2001 From: skyflow-bharti Date: Fri, 12 Sep 2025 19:28:07 +0530 Subject: [PATCH 04/10] SK-2293 update workflow --- ci-scripts/bump_version.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci-scripts/bump_version.sh b/ci-scripts/bump_version.sh index b0a57a9e..74fd983f 100755 --- a/ci-scripts/bump_version.sh +++ b/ci-scripts/bump_version.sh @@ -12,11 +12,11 @@ then echo "Done, Package now at $1" else - echo "Bumping package version to $1-dev.$2" + echo "Bumping package version to $1.dev0+$2" - sed -E "s/current_version = .+/current_version = \'$SEMVER-dev.$2\'/g" setup.py > tempfile && cat tempfile > setup.py && rm -f tempfile - sed -E "s/SDK_VERSION = .+/SDK_VERSION = \'$SEMVER-dev.$2\'/g" skyflow/version.py > tempfile && cat tempfile > skyflow/version.py && rm -f tempfile + sed -E "s/current_version = .+/current_version = \'$SEMVER.dev0+$2\'/g" setup.py > tempfile && cat tempfile > setup.py && rm -f tempfile + sed -E "s/SDK_VERSION = .+/SDK_VERSION = \'$SEMVER.dev0+$2\'/g" skyflow/version.py > tempfile && cat tempfile > skyflow/version.py && rm -f tempfile echo -------------------------- - echo "Done, Package now at $1-dev.$2" + echo "Done, Package now at $1.dev0+$2" fi From 221171fd600f6c415ae2febb405c0f67836e828e Mon Sep 17 00:00:00 2001 From: skyflow-bharti Date: Fri, 12 Sep 2025 13:58:40 +0000 Subject: [PATCH 05/10] [AUTOMATED] Private Release 2.1.0b1.dev0+8efd1cd --- setup.py | 2 +- skyflow/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 1739e441..6ae3bd78 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ if sys.version_info < (3, 7): raise RuntimeError("skyflow requires Python 3.7+") -current_version = '2.1.0b1-dev.f4e1cfd' +current_version = '2.1.0b1.dev0+8efd1cd' setup( name='skyflow', diff --git a/skyflow/version.py b/skyflow/version.py index 1ce30dae..70fdf1c4 100644 --- a/skyflow/version.py +++ b/skyflow/version.py @@ -1 +1 @@ -SDK_VERSION = '2.1.0b1-dev.f4e1cfd' \ No newline at end of file +SDK_VERSION = '2.1.0b1.dev0+8efd1cd' \ No newline at end of file From 373c6bb90c870c8fd1e14f54d2712614d9f1939e Mon Sep 17 00:00:00 2001 From: skyflow-shravan Date: Sun, 14 Sep 2025 18:59:50 +0530 Subject: [PATCH 06/10] SK-2285 insert with requests session --- skyflow/vault/_client.py | 52 +++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 24 deletions(-) diff --git a/skyflow/vault/_client.py b/skyflow/vault/_client.py index e21dcbab..5e1d7d81 100644 --- a/skyflow/vault/_client.py +++ b/skyflow/vault/_client.py @@ -5,6 +5,8 @@ import types import requests import asyncio +from requests.packages.urllib3.util.retry import Retry +from requests.adapters import HTTPAdapter from skyflow.vault._insert import getInsertRequestBody, processResponse, convertResponse from skyflow.vault._update import sendUpdateRequests, createUpdateResponseBody from skyflow.vault._config import Configuration, ConnectionConfig, DeleteOptions, DetokenizeOptions, GetOptions, InsertOptions, UpdateOptions, QueryOptions @@ -36,6 +38,16 @@ def __init__(self, config: Configuration): raise SkyflowError(SkyflowErrorCodes.INVALID_INPUT, SkyflowErrorMessages.TOKEN_PROVIDER_ERROR.value % ( str(type(config.tokenProvider))), interface=interface) + retry_strategy = Retry( + total=3, + backoff_factor=0.5, + status_forcelist=[500, 502, 503, 504], + ) + + self.session = requests.Session() + adapter = HTTPAdapter(pool_connections=1, pool_maxsize=20, pool_block=True, max_retries=retry_strategy) + self.session.mount("https://", adapter) + self.vaultID = config.vaultID self.vaultURL = config.vaultURL.rstrip('/') self.tokenProvider = config.tokenProvider @@ -55,30 +67,22 @@ def insert(self, records: dict, options: InsertOptions = InsertOptions()): "Authorization": "Bearer " + self.storedToken, "sky-metadata": json.dumps(getMetrics()) } - max_retries = 3 - # Use for-loop for retry logic, avoid code repetition - for attempt in range(max_retries+1): - try: - # If jsonBody is a dict, use json=, else use data= - response = requests.post(requestURL, data=jsonBody, headers=headers) - processedResponse = processResponse(response) - result, partial = convertResponse(records, processedResponse, options) - if partial: - log_error(SkyflowErrorMessages.BATCH_INSERT_PARTIAL_SUCCESS.value, interface) - raise SkyflowError(SkyflowErrorCodes.PARTIAL_SUCCESS, SkyflowErrorMessages.BATCH_INSERT_PARTIAL_SUCCESS.value, result, interface=interface) - if 'records' not in result: - log_error(SkyflowErrorMessages.BATCH_INSERT_FAILURE.value, interface) - raise SkyflowError(SkyflowErrorCodes.SERVER_ERROR, SkyflowErrorMessages.BATCH_INSERT_FAILURE.value, result, interface=interface) - log_info(InfoMessages.INSERT_DATA_SUCCESS.value, interface) - return result - except Exception as err: - if attempt < max_retries: - continue - else: - if isinstance(err, SkyflowError): - raise err - else: - raise SkyflowError(SkyflowErrorCodes.SERVER_ERROR, f"Error occurred: {err}", interface=interface) + # response = requests.post(requestURL, data=jsonBody, headers=headers) + response = self.session.post( + requestURL, + data=jsonBody, + headers=headers, + ) + processedResponse = processResponse(response) + print(">>> processedResponse", processedResponse) + result, partial = convertResponse(records, processedResponse, options) + if partial: + log_error(SkyflowErrorMessages.BATCH_INSERT_PARTIAL_SUCCESS.value, interface) + elif 'records' not in result: + log_error(SkyflowErrorMessages.BATCH_INSERT_FAILURE.value, interface) + else: + log_info(InfoMessages.INSERT_DATA_SUCCESS.value, interface) + return result def detokenize(self, records: dict, options: DetokenizeOptions = DetokenizeOptions()): interface = InterfaceName.DETOKENIZE.value From dbd4af2c6bb9f5ae807d2d7e1fb2721ed7af8bc9 Mon Sep 17 00:00:00 2001 From: skyflow-shravan Date: Mon, 15 Sep 2025 14:14:17 +0530 Subject: [PATCH 07/10] SK-2294 requests session --- skyflow/vault/_client.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/skyflow/vault/_client.py b/skyflow/vault/_client.py index 5e1d7d81..b728b7d7 100644 --- a/skyflow/vault/_client.py +++ b/skyflow/vault/_client.py @@ -40,12 +40,14 @@ def __init__(self, config: Configuration): retry_strategy = Retry( total=3, + connect=5, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504], + raise_on_status=True, ) self.session = requests.Session() - adapter = HTTPAdapter(pool_connections=1, pool_maxsize=20, pool_block=True, max_retries=retry_strategy) + adapter = HTTPAdapter(pool_connections=1, pool_maxsize=20, pool_block=False, max_retries=retry_strategy) self.session.mount("https://", adapter) self.vaultID = config.vaultID @@ -65,16 +67,18 @@ def insert(self, records: dict, options: InsertOptions = InsertOptions()): self.storedToken, self.tokenProvider, interface) headers = { "Authorization": "Bearer " + self.storedToken, - "sky-metadata": json.dumps(getMetrics()) + "sky-metadata": json.dumps(getMetrics()), + # "User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36' } # response = requests.post(requestURL, data=jsonBody, headers=headers) response = self.session.post( requestURL, data=jsonBody, headers=headers, + # timeout=(5, 300), ) processedResponse = processResponse(response) - print(">>> processedResponse", processedResponse) + print(">>> processedResponse local: ", processedResponse) result, partial = convertResponse(records, processedResponse, options) if partial: log_error(SkyflowErrorMessages.BATCH_INSERT_PARTIAL_SUCCESS.value, interface) From 6be388bdbcfda510b24e1ed95f5dd64f8bfa213f Mon Sep 17 00:00:00 2001 From: skyflow-bharti Date: Tue, 16 Sep 2025 13:02:53 +0530 Subject: [PATCH 08/10] SK-2293 check the response history --- skyflow/vault/_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/skyflow/vault/_client.py b/skyflow/vault/_client.py index b728b7d7..fb249b77 100644 --- a/skyflow/vault/_client.py +++ b/skyflow/vault/_client.py @@ -5,7 +5,7 @@ import types import requests import asyncio -from requests.packages.urllib3.util.retry import Retry +from urllib3.util.retry import Retry from requests.adapters import HTTPAdapter from skyflow.vault._insert import getInsertRequestBody, processResponse, convertResponse from skyflow.vault._update import sendUpdateRequests, createUpdateResponseBody @@ -77,6 +77,7 @@ def insert(self, records: dict, options: InsertOptions = InsertOptions()): headers=headers, # timeout=(5, 300), ) + print(">>> raw response: ", response.history) processedResponse = processResponse(response) print(">>> processedResponse local: ", processedResponse) result, partial = convertResponse(records, processedResponse, options) From 8a2b4e5bf0484453864025690956637076c74655 Mon Sep 17 00:00:00 2001 From: skyflow-bharti Date: Tue, 16 Sep 2025 07:35:17 +0000 Subject: [PATCH 09/10] [AUTOMATED] Private Release 1.15.4.dev0+ef26f0d --- setup.py | 2 +- skyflow/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 323fa31d..af42de74 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ if sys.version_info < (3, 7): raise RuntimeError("skyflow requires Python 3.7+") -current_version = '1.15.4' +current_version = '1.15.4.dev0+ef26f0d' setup( name='skyflow', diff --git a/skyflow/version.py b/skyflow/version.py index 9d9a8747..37d07012 100644 --- a/skyflow/version.py +++ b/skyflow/version.py @@ -1 +1 @@ -SDK_VERSION = '1.15.4' +SDK_VERSION = '1.15.4.dev0+ef26f0d' From 03930cc810ed90231de6fda485befd409cad97b5 Mon Sep 17 00:00:00 2001 From: skyflow-bharti Date: Tue, 16 Sep 2025 07:37:28 +0000 Subject: [PATCH 10/10] [AUTOMATED] Private Release 1.15.4.dev0+ccfb164 --- setup.py | 2 +- skyflow/version.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index af42de74..7c196387 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ if sys.version_info < (3, 7): raise RuntimeError("skyflow requires Python 3.7+") -current_version = '1.15.4.dev0+ef26f0d' +current_version = '1.15.4.dev0+ccfb164' setup( name='skyflow', diff --git a/skyflow/version.py b/skyflow/version.py index 37d07012..77d53965 100644 --- a/skyflow/version.py +++ b/skyflow/version.py @@ -1 +1 @@ -SDK_VERSION = '1.15.4.dev0+ef26f0d' +SDK_VERSION = '1.15.4.dev0+ccfb164'