diff --git a/.github/actions/docker-build-builder/action.yml b/.github/actions/docker-build-builder/action.yml index e818d9f7a9c2..2da1619023d2 100644 --- a/.github/actions/docker-build-builder/action.yml +++ b/.github/actions/docker-build-builder/action.yml @@ -26,7 +26,9 @@ runs: - name: Use client-python in OpenCTI from same branch shell: bash if: ${{ inputs.client_python_local == 'true' }} - run: sed -i 's|^pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti@${{ inputs.checkout_ref }}#subdirectory=client-python|' ./opencti-platform/opencti-graphql/src/python/requirements.txt + env: + CHECKOUT_REF: ${{ inputs.checkout_ref }} + run: sed -i 's|^pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti@'"$CHECKOUT_REF"'#subdirectory=client-python|' ./opencti-platform/opencti-graphql/src/python/requirements.txt - name: Set up Docker Buildx uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c #v4.2.0 diff --git a/.github/actions/docker-build-platform/action.yml b/.github/actions/docker-build-platform/action.yml index 2ddc08a19ae1..d74d85ff725b 100644 --- a/.github/actions/docker-build-platform/action.yml +++ b/.github/actions/docker-build-platform/action.yml @@ -26,7 +26,9 @@ runs: - name: Use client-python in OpenCTI from same branch shell: bash if: ${{ inputs.client_python_local == 'true' }} - run: sed -i 's|^pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti@${{ inputs.checkout_ref }}#subdirectory=client-python|' -i ./opencti-platform/opencti-graphql/src/python/requirements.txt + env: + CHECKOUT_REF: ${{ inputs.checkout_ref }} + run: sed -i 's|^pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti@'"$CHECKOUT_REF"'#subdirectory=client-python|' ./opencti-platform/opencti-graphql/src/python/requirements.txt - name: Set up Docker Buildx uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c #v4.2.0 diff --git a/.github/actions/docker-build-worker/action.yml b/.github/actions/docker-build-worker/action.yml index d8509d0a74e6..2470b69dee60 100644 --- a/.github/actions/docker-build-worker/action.yml +++ b/.github/actions/docker-build-worker/action.yml @@ -27,7 +27,9 @@ runs: - name: Use client-python in worker from same branch shell: bash if: ${{ inputs.client_python_local == 'true' }} - run: sed -i 's|^pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti@${{ inputs.checkout_ref }}#subdirectory=client-python|' -i ./opencti-worker/requirements.txt + env: + CHECKOUT_REF: ${{ inputs.checkout_ref }} + run: sed -i 's|^pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti@'"$CHECKOUT_REF"'#subdirectory=client-python|' -i ./opencti-worker/requirements.txt - name: Set up Docker Buildx uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c #v4.2.0 diff --git a/.github/workflows/cd-build-rolling-images-on-merge.yml b/.github/workflows/cd-build-rolling-images-on-merge.yml index 7f43859a7964..183ae90b785b 100644 --- a/.github/workflows/cd-build-rolling-images-on-merge.yml +++ b/.github/workflows/cd-build-rolling-images-on-merge.yml @@ -40,13 +40,18 @@ jobs: - name: Load builder image run: docker load --input /tmp/opencti-builder.tar + - name: Use client-python in OpenCTI from same branch + env: + CHECKOUT_REF: ${{ github.sha }} + run: sed -i 's|^pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti@'"$CHECKOUT_REF"'#subdirectory=client-python|' ./opencti-platform/opencti-graphql/src/python/requirements.txt + - name: Build graphql production and prune devDependencies run: | docker run --rm \ -v ${{ github.workspace }}/opencti-platform/opencti-graphql:/opt/opencti/opencti-graphql \ -w /opt/opencti/opencti-graphql \ filigran/builder:${{ github.sha }} \ - sh -c "yarn install --immutable && yarn install:python && yarn build:prod && yarn workspaces focus --production" + sh -c "yarn install --immutable && apk add --no-cache git && yarn install:python && yarn build:prod && yarn workspaces focus --production" - name: Build front production and prune devDependencies run: | diff --git a/.github/workflows/ci-docker-build.yml b/.github/workflows/ci-docker-build.yml index 30e81d9d84d4..5baa98ee83e5 100644 --- a/.github/workflows/ci-docker-build.yml +++ b/.github/workflows/ci-docker-build.yml @@ -22,6 +22,13 @@ on: type: string required: false default: false + secrets: + DOCKERHUB_USERNAME: + description: "Docker Hub username, required when publish_to_registry is true" + required: false + DOCKERHUB_TOKEN: + description: "Docker Hub access token, required when publish_to_registry is true" + required: false outputs: image-tag-version-cleaned: description: "Image tag after formatting" @@ -45,7 +52,9 @@ jobs: - name: Use client-python in OpenCTI from same branch if: ${{ inputs.client_python_local == true }} - run: sed -i 's|^pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti@${{ inputs.checkout_ref }}#subdirectory=client-python|' -i ./opencti-platform/opencti-graphql/src/python/requirements.txt + env: + CHECKOUT_REF: ${{ inputs.checkout_ref }} + run: sed -i 's|^pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti@'"$CHECKOUT_REF"'#subdirectory=client-python|' -i ./opencti-platform/opencti-graphql/src/python/requirements.txt - name: Set up Docker Buildx uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c #v4.2.0 @@ -122,7 +131,9 @@ jobs: - name: Use client-python in worker from same branch if: ${{ inputs.client_python_local == true }} - run: sed -i 's|^pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti@${{ inputs.checkout_ref }}#subdirectory=client-python|' -i ./opencti-worker/requirements.txt + env: + CHECKOUT_REF: ${{ inputs.checkout_ref }} + run: sed -i 's|^pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti@'"$CHECKOUT_REF"'#subdirectory=client-python|' -i ./opencti-worker/requirements.txt - name: Set up Docker Buildx uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c #v4.2.0 diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml index 3b9bbf6f2f7a..becbd9dc09b9 100644 --- a/.github/workflows/ci-main.yml +++ b/.github/workflows/ci-main.yml @@ -50,15 +50,18 @@ jobs: needs: [wf-build-platform-image,wf-build-worker-image,wf-build-builder-image] with: checkout_ref: ${{ github.event_name == 'pull_request' && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }} + client_python_local: true run-id: ${{ github.run_id }} - secrets: inherit + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} wf-frontend-quality: name: Frontend quality uses: ./.github/workflows/ci-test-frontend-quality.yml with: checkout_ref: ${{ github.event_name == 'pull_request' && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }} - secrets: inherit + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} wf-frontend-test: name: Frontend e2e @@ -67,7 +70,6 @@ jobs: with: checkout_ref: ${{ github.event_name == 'pull_request' && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }} run-id: ${{ github.run_id }} - secrets: inherit wf-client-python-test: @@ -76,11 +78,11 @@ jobs: needs: [wf-build-platform-image,wf-build-worker-image] with: checkout_ref: ${{ github.event_name == 'pull_request' && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }} - secrets: inherit + secrets: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} wf-license-check: name: License check uses: ./.github/workflows/ci-license-check.yml with: checkout_ref: ${{ github.event_name == 'pull_request' && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }} - secrets: inherit \ No newline at end of file diff --git a/.github/workflows/ci-test-backend.yml b/.github/workflows/ci-test-backend.yml index 970d2b97fce7..3e0c68f4680f 100644 --- a/.github/workflows/ci-test-backend.yml +++ b/.github/workflows/ci-test-backend.yml @@ -7,9 +7,18 @@ on: description: "Reference of the branch or commit sha" type: string default: ${{ github.sha }} + client_python_local: + description: "Use client python from the same commit" + type: boolean + required: false + default: true run-id: required: true type: string + secrets: + CODECOV_TOKEN: + description: "Token used to upload coverage reports to Codecov" + required: false jobs: run-opencti-integration-test: @@ -26,6 +35,12 @@ jobs: continue-on-error: true run: ./scripts/ci/runner-info.sh # Display runner info and start resource monitoring in background + - name: Use client-python in OpenCTI from same branch + if: ${{ inputs.client_python_local == true }} + env: + CHECKOUT_REF: ${{ inputs.checkout_ref }} + run: sed -i 's|^pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti@'"$CHECKOUT_REF"'#subdirectory=client-python|' ./opencti-platform/opencti-graphql/src/python/requirements.txt + - name: Fetch EE license run: | echo "EE_LICENSE=$(wget -qO- https://filigran.jfrog.io/artifactory/ci-utils/opencti-license/opencti_ee_ci_licence.crt | tr -d '\n')" >> "$GITHUB_ENV" @@ -97,6 +112,7 @@ jobs: mkdir -p ../../opencti-connectors/external-import/restore-files/src wget -O ../../opencti-connectors/external-import/restore-files/src/restore-files.py https://raw.githubusercontent.com/OpenCTI-Platform/connectors/master/external-import/restore-files/src/restore-files.py yarn install + apk add --no-cache git # required by pip to install pycti from a git reference yarn install:python yarn build NODE_OPTIONS=--max_old_space_size=8192 yarn test:ci-integration-sync --coverage @@ -169,11 +185,19 @@ jobs: steps: - name: OpenCTI checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout_ref }} - name: Runner information continue-on-error: true run: ./scripts/ci/runner-info.sh # Display runner info and start resource monitoring in background + - name: Use client-python in OpenCTI from same branch + if: ${{ inputs.client_python_local == true }} + env: + CHECKOUT_REF: ${{ inputs.checkout_ref }} + run: sed -i 's|^pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti@'"$CHECKOUT_REF"'#subdirectory=client-python|' ./opencti-platform/opencti-graphql/src/python/requirements.txt + - name: Fetch EE license run: | echo "EE_LICENSE=$(wget -qO- https://filigran.jfrog.io/artifactory/ci-utils/opencti-license/opencti_ee_ci_licence.crt | tr -d '\n')" >> "$GITHUB_ENV" @@ -220,13 +244,13 @@ jobs: -e "APP__ENTERPRISE_EDITION_LICENSE=${{ env.EE_LICENSE }}" \ filigran/builder:${{ github.sha }} \ sh -c 'set -eux - cd /home/workspace/opencti-platform/opencti-graphql mkdir -p ../../opencti-connectors/stream/backup-files/src wget -O ../../opencti-connectors/stream/backup-files/src/backup-files.py https://raw.githubusercontent.com/OpenCTI-Platform/connectors/master/stream/backup-files/src/backup-files.py mkdir -p ../../opencti-connectors/external-import/restore-files/src wget -O ../../opencti-connectors/external-import/restore-files/src/restore-files.py https://raw.githubusercontent.com/OpenCTI-Platform/connectors/master/external-import/restore-files/src/restore-files.py yarn install + apk add --no-cache git # required by pip to install pycti from a git reference yarn install:python yarn get-connectors-manifest NODE_OPTIONS=--max_old_space_size=8192 yarn test:ci-rules-and-others --coverage @@ -269,11 +293,19 @@ jobs: steps: - name: OpenCTI checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout_ref }} - name: Runner information continue-on-error: true run: ./scripts/ci/runner-info.sh # Display runner info and start resource monitoring in background + - name: Use client-python in OpenCTI from same branch + if: ${{ inputs.client_python_local == true }} + env: + CHECKOUT_REF: ${{ inputs.checkout_ref }} + run: sed -i 's|^pycti==.*$|pycti @ git+https://github.com/OpenCTI-Platform/opencti@'"$CHECKOUT_REF"'#subdirectory=client-python|' ./opencti-platform/opencti-graphql/src/python/requirements.txt + - name: Download image builder continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c #v8.0.1 @@ -297,6 +329,7 @@ jobs: sh -c 'set -eux cd /home/workspace/opencti-platform/opencti-graphql yarn install + apk add --no-cache git # required by pip to install pycti from a git reference yarn install:python yarn build ' @@ -324,6 +357,7 @@ jobs: filigran/builder:${{ github.sha }} \ sh -c 'set -eux cd /home/workspace/opencti-platform/opencti-graphql + apk add --no-cache git # required by pip to install pycti from a git reference yarn install:python NODE_OPTIONS=--max_old_space_size=8192 yarn test:ci-unit --coverage ' diff --git a/.github/workflows/ci-test-client-python.yml b/.github/workflows/ci-test-client-python.yml index a2c0f4778cf5..a57120ef4cf4 100644 --- a/.github/workflows/ci-test-client-python.yml +++ b/.github/workflows/ci-test-client-python.yml @@ -7,6 +7,10 @@ on: description: "Reference of the branch or commit sha" type: string default: ${{ github.sha }} + secrets: + CODECOV_TOKEN: + description: "Token used to upload coverage reports to Codecov" + required: false jobs: ensure-formatting-pycti: diff --git a/.github/workflows/ci-test-frontend-quality.yml b/.github/workflows/ci-test-frontend-quality.yml index e47720f462b2..e9d99dec5bb6 100644 --- a/.github/workflows/ci-test-frontend-quality.yml +++ b/.github/workflows/ci-test-frontend-quality.yml @@ -7,6 +7,10 @@ on: description: "Reference of the branch or commit sha" type: string default: ${{ github.sha }} + secrets: + CODECOV_TOKEN: + description: "Token used to upload coverage reports to Codecov" + required: false jobs: opencti-frontend-test: diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 862ab26d124d..e12f0772e761 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -37,13 +37,13 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Initialize CodeQL - uses: github/codeql-action/init@v4 + uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: languages: ${{ matrix.language }} config-file: ./.github/codeql-config.yml - name: Autobuild - uses: github/codeql-action/autobuild@v4 + uses: github/codeql-action/autobuild@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4 + uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 diff --git a/.github/workflows/deploy-testing-xtm-one.yml b/.github/workflows/deploy-testing-xtm-one.yml index bc0a72cf271e..27bd18985839 100644 --- a/.github/workflows/deploy-testing-xtm-one.yml +++ b/.github/workflows/deploy-testing-xtm-one.yml @@ -17,7 +17,9 @@ jobs: image_tag: testing-xtm-one client_python_local: true publish_to_registry: true - secrets: inherit + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} wf-redeploy-instance: name: Restart testing-xtm-one instance diff --git a/.github/workflows/test-feature-branch.yml b/.github/workflows/test-feature-branch.yml index f6b06d67fb92..2f15cc142338 100644 --- a/.github/workflows/test-feature-branch.yml +++ b/.github/workflows/test-feature-branch.yml @@ -31,7 +31,9 @@ jobs: image_tag: ${{ github.ref_name }} client_python_local: ${{ inputs.client_python_local }} publish_to_registry: true - secrets: inherit + secrets: + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} deploy: name: deploy