SK-2292 internal release for v1

.github/workflows/internal_release.yml

@@ -0,0 +1,23 @@
+name: Internal Release
+
+on:
+  push:
+    tags-ignore:
+      - '*.*'
+    paths-ignore:
+      - "setup.py"
+      - "*.yml"
+      - "*.md"
+      - "skyflow/version.py"
+      - "samples/**"
+    branches:
+      - release/*
+
+jobs:
+  build-and-deploy:
+    uses: ./.github/workflows/shared-build-and-deploy.yml
+    with:
+      ref: ${{ github.ref_name }}
+      tag: 'internal'
+    secrets: inherit
+

.github/workflows/shared-build-and-deploy.yml

@@ -0,0 +1,83 @@
+name: Shared Build and Deploy
+
+on:
+  workflow_call:
+    inputs:
+      ref:
+        description: 'Git reference to use (e.g., main or branch name)'
+        required: true
+        type: string
+
+      tag:
+        description: 'Release Tag'
+        required: true
+        type: string
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v2
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+
+      - name: Resolve Branch for the Tagged Commit
+        id: resolve-branch
+        if: ${{ inputs.tag == 'beta' || inputs.tag == 'public' }} 
+        run: |
+          TAG_COMMIT=$(git rev-list -n 1 ${{ github.ref_name }})
+
+          BRANCH_NAME=$(git branch -r --contains $TAG_COMMIT | grep -o 'origin/.*' | sed 's|origin/||' | head -n 1)
+
+          if [ -z "$BRANCH_NAME" ]; then
+            echo "Error: Could not resolve branch for the tag."
+            exit 1
+          fi
+
+          echo "Resolved Branch Name: $BRANCH_NAME"
+          echo "branch_name=$BRANCH_NAME" >> $GITHUB_ENV
+
+      - name: Get Previous tag
+        id: previoustag
+        uses: WyriHaximus/github-action-get-previous-tag@v1
+        with:
+          fallback: 1.0.0
+
+      - name: Bump Version
+        run: |
+          chmod +x ./ci-scripts/bump_version.sh
+          if ${{ inputs.tag == 'internal' }}; then
+            ./ci-scripts/bump_version.sh "${{ steps.previoustag.outputs.tag }}" "$(git rev-parse --short "$GITHUB_SHA")"
+          else
+            ./ci-scripts/bump_version.sh "${{ steps.previoustag.outputs.tag }}"
+          fi
+
+      - name: Commit changes
+        run: |
+          git config user.name "${{ github.actor }}"
+          git config user.email "${{ github.actor }}@users.noreply.github.com"
+
+          git add setup.py
+          git add skyflow/version.py
+
+          if [[ "${{ inputs.tag }}" == "internal" ]]; then
+            VERSION="${{ steps.previoustag.outputs.tag }}.dev0+$(git rev-parse --short $GITHUB_SHA)"
+            COMMIT_MESSAGE="[AUTOMATED] Private Release $VERSION"
+            git commit -m "$COMMIT_MESSAGE"
+            git push origin ${{ github.ref_name }} -f
+          fi
+
+      - name: Build and Publish to JFrog Artifactory
+        if: ${{ inputs.tag == 'internal' }}
+        env:
+          TWINE_USERNAME: ${{ secrets.JFROG_USERNAME }}
+          TWINE_PASSWORD: ${{ secrets.JFROG_PASSWORD }}
+        run: |
+          python setup.py sdist bdist_wheel
+          twine upload --repository-url https://prekarilabs.jfrog.io/artifactory/api/pypi/skyflow-python/ dist/*

ci-scripts/bump_version.sh

@@ -12,11 +12,11 @@ then
 	echo "Done, Package now at $1"
 
 else
-	echo "Bumping package version to $1-dev.$2"
+	echo "Bumping package version to $1.dev0+$2"
 
-	sed -E "s/current_version = .+/current_version = \'$SEMVER-dev.$2\'/g" setup.py > tempfile && cat tempfile > setup.py && rm -f tempfile
-	sed -E "s/SDK_VERSION = .+/SDK_VERSION = \'$SEMVER-dev.$2\'/g" skyflow/version.py > tempfile && cat tempfile > skyflow/version.py && rm -f tempfile
+	sed -E "s/current_version = .+/current_version = \'$SEMVER.dev0+$2\'/g" setup.py > tempfile && cat tempfile > setup.py && rm -f tempfile
+	sed -E "s/SDK_VERSION = .+/SDK_VERSION = \'$SEMVER.dev0+$2\'/g" skyflow/version.py > tempfile && cat tempfile > skyflow/version.py && rm -f tempfile
 
 	echo --------------------------
-	echo "Done, Package now at $1-dev.$2"
+	echo "Done, Package now at $1.dev0+$2"
 fi

setup.py

@@ -6,7 +6,7 @@
 
 if sys.version_info < (3, 7):
     raise RuntimeError("skyflow requires Python 3.7+")
-current_version = '1.15.4'
+current_version = '1.15.4.dev0+ccfb164'
 
 setup(
     name='skyflow',

skyflow/vault/_client.py

@@ -5,6 +5,8 @@
 import types
 import requests
 import asyncio
+from urllib3.util.retry import Retry
+from requests.adapters import HTTPAdapter
 from skyflow.vault._insert import getInsertRequestBody, processResponse, convertResponse
 from skyflow.vault._update import sendUpdateRequests, createUpdateResponseBody
 from skyflow.vault._config import Configuration, ConnectionConfig, DeleteOptions, DetokenizeOptions, GetOptions, InsertOptions, UpdateOptions, QueryOptions
@@ -36,6 +38,18 @@ def __init__(self, config: Configuration):
             raise SkyflowError(SkyflowErrorCodes.INVALID_INPUT, SkyflowErrorMessages.TOKEN_PROVIDER_ERROR.value % (
                 str(type(config.tokenProvider))), interface=interface)
 
+        retry_strategy = Retry(
+            total=3,
+            connect=5,
+            backoff_factor=0.5,
+            status_forcelist=[500, 502, 503, 504],
+            raise_on_status=True,
+        )
+
+        self.session = requests.Session()
+        adapter = HTTPAdapter(pool_connections=1, pool_maxsize=20, pool_block=False, max_retries=retry_strategy)
+        self.session.mount("https://", adapter)
+
         self.vaultID = config.vaultID
         self.vaultURL = config.vaultURL.rstrip('/')
         self.tokenProvider = config.tokenProvider
@@ -53,32 +67,27 @@ def insert(self, records: dict, options: InsertOptions = InsertOptions()):
             self.storedToken, self.tokenProvider, interface)
         headers = {
             "Authorization": "Bearer " + self.storedToken,
-            "sky-metadata": json.dumps(getMetrics())
+            "sky-metadata": json.dumps(getMetrics()),
+            # "User-Agent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'
         }
-        max_retries = 3
-        # Use for-loop for retry logic, avoid code repetition
-        for attempt in range(max_retries+1):
-            try:
-                # If jsonBody is a dict, use json=, else use data=
-                response = requests.post(requestURL, data=jsonBody, headers=headers)
-                processedResponse = processResponse(response)
-                result, partial = convertResponse(records, processedResponse, options)
-                if partial:
-                    log_error(SkyflowErrorMessages.BATCH_INSERT_PARTIAL_SUCCESS.value, interface)
-                    raise SkyflowError(SkyflowErrorCodes.PARTIAL_SUCCESS, SkyflowErrorMessages.BATCH_INSERT_PARTIAL_SUCCESS.value, result, interface=interface)
-                if 'records' not in result:
-                    log_error(SkyflowErrorMessages.BATCH_INSERT_FAILURE.value, interface)
-                    raise SkyflowError(SkyflowErrorCodes.SERVER_ERROR, SkyflowErrorMessages.BATCH_INSERT_FAILURE.value, result, interface=interface)
-                log_info(InfoMessages.INSERT_DATA_SUCCESS.value, interface)
-                return result
-            except Exception as err:
-                if attempt < max_retries:
-                    continue
-                else:
-                    if isinstance(err, SkyflowError):
-                        raise err
-                    else:
-                        raise SkyflowError(SkyflowErrorCodes.SERVER_ERROR, f"Error occurred: {err}", interface=interface)
+        # response = requests.post(requestURL, data=jsonBody, headers=headers)
+        response = self.session.post(
+            requestURL,
+            data=jsonBody,
+            headers=headers,
+            # timeout=(5, 300),
+        )
+        print(">>> raw response: ", response.history)
+        processedResponse = processResponse(response)
+        print(">>> processedResponse local: ", processedResponse)
+        result, partial = convertResponse(records, processedResponse, options)
+        if partial:
+            log_error(SkyflowErrorMessages.BATCH_INSERT_PARTIAL_SUCCESS.value, interface)
+        elif 'records' not in result:
+            log_error(SkyflowErrorMessages.BATCH_INSERT_FAILURE.value, interface)
+        else:
+            log_info(InfoMessages.INSERT_DATA_SUCCESS.value, interface)
+        return result
 
     def detokenize(self, records: dict, options: DetokenizeOptions = DetokenizeOptions()):
         interface = InterfaceName.DETOKENIZE.value

skyflow/version.py

@@ -1 +1 @@
-SDK_VERSION = '1.15.4'
+SDK_VERSION = '1.15.4.dev0+ccfb164'