Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
182 changes: 174 additions & 8 deletions .github/workflows/dotnet-tool.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,21 @@
# Publishes the Windows Thrift compiler to NuGet as a .NET tool, so that it can
# be installed with "dotnet tool install --global Apache.Thrift.Compiler".
#
# The compiler build mirrors the compiler-windows job in cmake.yml, which is
# what actually tests it on every push.
# The package of a release has to carry the thrift.exe the vote covered, not a
# new build of the same source: two builds are not the same bytes. So where
# there is such an executable, this workflow packs that one, as
# windows-packages.yml does for the installer:
#
# - for a GitHub release, the thrift-<version>.exe in the release's
# directory on dist.apache.org;
# - for a manual run with compiler_url, the executable at that URL, such as
# the one a release candidate put on dist/dev. A manual run never
# publishes; it shows that the package works on a Windows runner.
#
# It is used only once its checksum and its signature against KEYS check out.
# Pull requests, and manual runs without compiler_url, build the compiler from
# source instead. That build mirrors the compiler-windows job in cmake.yml,
# which is what actually tests it on every push.

name: ".NET tool"

Expand All @@ -37,6 +50,11 @@ on:
- "build/windows/check-compiler-imports.ps1"
- "doc/ReleaseManagement.md"
workflow_dispatch:
inputs:
compiler_url:
description: "A signed thrift-X.Y.Z.exe on dist.apache.org to pack, for example https://dist.apache.org/repos/dist/release/thrift/0.25.0/thrift-0.25.0.exe. Empty builds the compiler from the branch instead. A manual run never publishes."
required: false
type: string

concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
Expand All @@ -46,8 +64,84 @@ permissions:
contents: read

jobs:
voted-compiler:
name: Voted compiler
if: ${{ github.event_name == 'release' || inputs.compiler_url != '' }}
# gpg is at hand on Linux.
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
outputs:
version: ${{ steps.fetch.outputs.version }}
url: ${{ steps.fetch.outputs.url }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false

# windows-packages.yml runs these on every pull request that touches
# them. Here they run before the download they guard, so that a release
# does not rely on checks that no longer work.
- name: Test the download checks
shell: pwsh
run: ./build/windows/get-voted-compiler-tests.ps1

- name: Download and check the compiler
id: fetch
shell: pwsh
env:
RELEASE_TAG: ${{ github.event.release.tag_name }}
INPUT_URL: ${{ inputs.compiler_url }}
run: |
$url = $env:INPUT_URL
if (-not $url) {
# A release is promoted to dist/release before its GitHub release
# is published, so the executable is already there.
$version = $env:RELEASE_TAG -replace '^v', ''
$url = "https://dist.apache.org/repos/dist/release/thrift/$version/thrift-$version.exe"
}
"url=$url" | Out-File -FilePath $env:GITHUB_OUTPUT -Append
./build/windows/get-voted-compiler.ps1 -Url $url -OutputDir voted
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

- name: Collect LICENSE and NOTICE
env:
VERSION: ${{ steps.fetch.outputs.version }}
run: |
# The package carries the LICENSE and NOTICE of the release it
# packs. A release has a tag to take them from. A release
# candidate has none yet, so its files come from the branch this
# runs on, which then has to be at that version.
if git fetch --quiet --depth=1 origin "refs/tags/v${VERSION}:refs/tags/v${VERSION}" 2>/dev/null; then
git show "v${VERSION}:LICENSE" > voted/LICENSE
git show "v${VERSION}:NOTICE" > voted/NOTICE
echo "LICENSE and NOTICE from tag v${VERSION}"
else
branch_version=$(sed -n 's/^ *set(thrift_VERSION "\([0-9.]*\)").*/\1/p' CMakeLists.txt)
if [ "$branch_version" != "$VERSION" ]; then
echo "::error::There is no tag v${VERSION}, and this branch is at ${branch_version}. Run this from the branch the release candidate was made from."
exit 1
fi
cp LICENSE NOTICE voted/
echo "LICENSE and NOTICE from ${GITHUB_REF_NAME}"
fi

- name: Archive the compiler
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: voted-compiler
if-no-files-found: error
path: voted/
retention-days: 3

pack:
name: Pack
needs: voted-compiler
# Runs when the voted compiler is there, and when there was none to fetch
# and voted-compiler was skipped. Not when fetching or checking it failed:
# a release must not fall back to building its own compiler.
if: ${{ !cancelled() && needs.voted-compiler.result != 'failure' }}
runs-on: windows-2025
timeout-minutes: 60
permissions:
Expand All @@ -64,6 +158,7 @@ jobs:
shell: pwsh
env:
RELEASE_TAG: ${{ github.event.release.tag_name }}
VOTED_VERSION: ${{ needs.voted-compiler.outputs.version }}
run: |
$sourceVersion = ''
foreach ($line in Get-Content CMakeLists.txt) {
Expand All @@ -90,16 +185,36 @@ jobs:
$version = $tagVersion
}

if ($env:VOTED_VERSION) {
# On a release the voted executable was fetched for the tag's
# version. A manual run packs whatever release compiler_url names,
# which need not be the one this branch is at.
if ($env:RELEASE_TAG -and $env:VOTED_VERSION -ne $version) {
Write-Error "Version mismatch: voted compiler=$env:VOTED_VERSION, tag=$env:RELEASE_TAG"
exit 1
}
$version = $env:VOTED_VERSION
}

Write-Host "Version: $version"
"version=$version" | Out-File -FilePath $env:GITHUB_OUTPUT -Append

- name: Download the voted compiler
if: ${{ needs.voted-compiler.result == 'success' }}
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: voted-compiler
path: voted

- name: Install dependencies
if: ${{ needs.voted-compiler.result != 'success' }}
shell: pwsh
timeout-minutes: 10
# Pinned to the version build/docker/msvc/Dockerfile uses.
run: choco install winflexbison3 --version=2.5.24.20210105 -y --no-progress

- name: Build the compiler
if: ${{ needs.voted-compiler.result != 'success' }}
shell: pwsh
timeout-minutes: 30
run: |
Expand All @@ -110,32 +225,83 @@ jobs:
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
cmake --install cmake_build --config Release --prefix cmake_install

- name: Choose the compiler to pack
id: compiler
shell: pwsh
env:
VERSION: ${{ steps.version.outputs.version }}
VOTED: ${{ needs.voted-compiler.result == 'success' }}
run: |
if ($env:VOTED -eq 'true') {
$compiler = "voted/thrift-$env:VERSION.exe"
# The LICENSE and NOTICE of that release, collected with it.
$sourceRoot = 'voted'
}
else {
$compiler = 'cmake_install/bin/thrift.exe'
$sourceRoot = ''
}
$sha256 = (Get-FileHash -Algorithm SHA256 -LiteralPath $compiler).Hash.ToLowerInvariant()
Write-Host "Packing $compiler, sha256 $sha256"
"path=$compiler" | Out-File -FilePath $env:GITHUB_OUTPUT -Append
"source_root=$sourceRoot" | Out-File -FilePath $env:GITHUB_OUTPUT -Append
"sha256=$sha256" | Out-File -FilePath $env:GITHUB_OUTPUT -Append

- name: Check what the compiler imports
shell: pwsh
env:
COMPILER: ${{ steps.compiler.outputs.path }}
# A published NuGet version cannot be replaced, so this has to hold
# before the package is built, not only in cmake.yml afterwards.
run: ./build/windows/check-compiler-imports.ps1 -Path cmake_install/bin/thrift.exe
run: ./build/windows/check-compiler-imports.ps1 -Path $env:COMPILER

- name: Pack the tool
shell: pwsh
env:
VERSION: ${{ steps.version.outputs.version }}
COMPILER: ${{ steps.compiler.outputs.path }}
SOURCE_ROOT: ${{ steps.compiler.outputs.source_root }}
run: |
./build/windows/build-dotnet-tool.ps1 `
-Version $env:VERSION `
-Compiler cmake_install/bin/thrift.exe `
-OutputDir nupkg
$toolArgs = @{ Version = $env:VERSION; Compiler = $env:COMPILER; OutputDir = 'nupkg' }
if ($env:SOURCE_ROOT) {
$toolArgs['SourceRoot'] = $env:SOURCE_ROOT
}
./build/windows/build-dotnet-tool.ps1 @toolArgs

- name: Test the package
shell: pwsh
timeout-minutes: 15
env:
VERSION: ${{ steps.version.outputs.version }}
COMPILER_SHA256: ${{ steps.compiler.outputs.sha256 }}
run: |
# The hash shows that the package carries the very file it was
# given - for a release, the one the vote covered.
./build/windows/dotnet-tool/test-dotnet-tool.ps1 `
-Package "nupkg/Apache.Thrift.Compiler.$env:VERSION.nupkg" `
-Version $env:VERSION `
-ExpectedCompilerOutput "Thrift version $env:VERSION"
-ExpectedCompilerOutput "Thrift version $env:VERSION" `
-ExpectedSha256 $env:COMPILER_SHA256

- name: Report the compiler
shell: pwsh
env:
VERSION: ${{ steps.version.outputs.version }}
COMPILER_SHA256: ${{ steps.compiler.outputs.sha256 }}
COMPILER_URL: ${{ needs.voted-compiler.outputs.url }}
run: |
$package = "nupkg/Apache.Thrift.Compiler.$env:VERSION.nupkg"
$hash = (Get-FileHash -Algorithm SHA256 $package).Hash.ToLowerInvariant()
@(
'### .NET tool',
'',
"| | |",
"|---|---|",
"| package | ``$([System.IO.Path]::GetFileName($package))`` |",
"| package sha256 | ``$hash`` |",
"| compiler | $(if ($env:COMPILER_URL) { $env:COMPILER_URL } else { 'built from source' }) |",
"| compiler sha256 | ``$env:COMPILER_SHA256`` |"
) | Out-File -FilePath $env:GITHUB_STEP_SUMMARY -Append

- name: Archive the package
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
Expand Down
23 changes: 16 additions & 7 deletions build/windows/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -78,11 +78,12 @@ are accepted.
PS C:\thrift> .\build\windows\get-voted-compiler.ps1 -Url https://dist.apache.org/repos/dist/release/thrift/0.25.0/thrift-0.25.0.exe -OutputDir voted
```

The installer of a release has to contain the executable the vote covered, and
a new build of the same source is not the same bytes. The
[`Windows packages`](../../.github/workflows/windows-packages.yml) workflow uses
this to get that executable. `-Path` and `-KeysPath` run the same checks on
files that are already local. It needs `gpg`.
The installer and the .NET tool of a release have to contain the executable the
vote covered, and a new build of the same source is not the same bytes. The
[`Windows packages`](../../.github/workflows/windows-packages.yml) and
[`.NET tool`](../../.github/workflows/dotnet-tool.yml) workflows use this to get
that executable. `-Path` and `-KeysPath` run the same checks on files that are
already local. It needs `gpg`.

## `get-voted-compiler-tests.ps1`

Expand Down Expand Up @@ -164,7 +165,14 @@ PS C:\thrift> .\build\windows\build-dotnet-tool.ps1 -Version 0.26.0 -Compiler C:
As with the installer, the version is passed in rather than stored in the
project file, so [`build/veralign.sh`](../veralign.sh) gains nothing to keep in
step. Building without `-Compiler` fails rather than producing a package with no
compiler in it.
compiler in it. `LICENSE` and `NOTICE` come from the checkout unless
`-SourceRoot` names another directory.

For a release, the [`.NET tool`](../../.github/workflows/dotnet-tool.yml)
workflow packs the voted `thrift-<version>.exe` from `dist/release`, fetched
and checked with `get-voted-compiler.ps1`, together with the `LICENSE` and
`NOTICE` of the release tag. Started by hand with `compiler_url`, it does the
same with the executable at that URL and publishes nothing.

## `dotnet-tool/`

Expand Down Expand Up @@ -199,7 +207,8 @@ in there and are not empty - and then proves the installed command works.

It runs anywhere PowerShell and the .NET SDK do. Off Windows it checks the
launcher's refusal instead of the compiler's output, which is the behaviour that
keeps a Linux user from a confusing failure.
keeps a Linux user from a confusing failure. `-ExpectedSha256` checks that the
bundled `thrift.exe` is the very file the package was built from.

## `build-chocolatey-package.ps1` and `chocolatey/`

Expand Down
9 changes: 6 additions & 3 deletions build/windows/dotnet-tool/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,9 @@ distributions package Thrift, and Homebrew has it as `thrift`.

## Provenance

The executable in this package is built from the Apache Thrift source release of
the same version. Releases are voted on by the Apache Thrift PMC; the source
release is the official artifact, and this package is a convenience build of it.
The executable in this package is the Windows compiler that the Apache Thrift
PMC voted on with the source release of the same version, the
`thrift-<version>.exe` on the [Download](https://thrift.apache.org/download)
page. Its checksums and its signature are checked against the project's `KEYS`
before it is packed. The source release is the official artifact, and this
package is a convenience package of it.
26 changes: 25 additions & 1 deletion build/windows/dotnet-tool/test-dotnet-tool.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,11 @@
Text the bundled executable should print when run. Only checked on
Windows. Optional.

.PARAMETER ExpectedSha256
The SHA-256 the bundled thrift.exe should have, to show that the package
carries the very file it was built from - for a release, the one the vote
covered. Checked on every platform. Optional.

.EXAMPLE
pwsh build\windows\dotnet-tool\test-dotnet-tool.ps1 -Package nupkg\Apache.Thrift.Compiler.0.26.0.nupkg -Version 0.26.0
#>
Expand All @@ -54,7 +59,8 @@ param(
[string] $Package,
[Parameter(Mandatory = $true)]
[string] $Version,
[string] $ExpectedCompilerOutput = ''
[string] $ExpectedCompilerOutput = '',
[string] $ExpectedSha256 = ''
)

$ErrorActionPreference = 'Stop'
Expand Down Expand Up @@ -117,6 +123,24 @@ try {
Assert-True 'the compiler appears exactly once' ($payloads.Count -eq 1) `
("found: " + ($payloads -join ', '))

if ($ExpectedSha256) {
$entry = $archive.GetEntry("$toolDir/thrift.exe")
$actual = ''
if ($entry) {
$stream = $entry.Open()
$sha = [System.Security.Cryptography.SHA256]::Create()
try {
$actual = [System.BitConverter]::ToString($sha.ComputeHash($stream)).Replace('-', '').ToLowerInvariant()
}
finally {
$sha.Dispose()
$stream.Dispose()
}
}
Assert-True 'the bundled compiler is the expected file' `
($actual -eq $ExpectedSha256.ToLowerInvariant()) "sha256 $actual, expected $ExpectedSha256"
}

function Read-Entry {
param([string] $Name)

Expand Down
5 changes: 5 additions & 0 deletions doc/ReleaseManagement.md
Original file line number Diff line number Diff line change
Expand Up @@ -619,6 +619,11 @@ See https://thrift.apache.org/lib/ for the current status of each external packa
can be installed with `dotnet tool install --global Apache.Thrift.Compiler`. It is not the
`ApacheThrift` runtime library, which is still published by hand following the instructions in
`ApacheThrift.nuspec`.
* The package carries the voted `thrift-<version>.exe` from `dist/release`, not a new build.
The workflow downloads it and checks its checksums and its signature against `KEYS` first;
if that fails, nothing is packed or published. To check the package beforehand, for example
around a release candidate's executable, start the workflow from the Actions tab with
`compiler_url` set to that executable on `dist.apache.org`. A manual run never publishes.
* Before publishing, verify that nuget.org has a trusted publishing policy for the
`apache/thrift` repository, the `.github/workflows/dotnet-tool.yml` workflow and the
`release` environment, and that the repository variable `NUGET_USER` names the nuget.org
Expand Down
Loading