Add tarball git-tree-sha1 metadata

Co-authored-by: OpenAI Codex GPT-5 <codex@openai.com>

Author: DilumAluthge-LLM

Project.toml

@@ -10,6 +10,7 @@ JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 Lazy = "50d2b5c4-7a5e-59d5-8109-a42b560f39c0"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
 TimeZones = "f269a46b-ccf7-5d73-abea-4c690281aa53"
 WebCacheUtilities = "0c1c26de-fc5f-47ff-87a8-a157289a9bac"
 

schema.json

@@ -41,6 +41,9 @@
                 "sha256": {
                     "type": "string"
                 },
+                "git-tree-sha1": {
+                    "type": "string"
+                },
                 "size": {
                     "type": "integer"
                 },

src/VersionsJSONUtil.jl

@@ -1,7 +1,8 @@
 module VersionsJSONUtil
 
-using HTTP, JSON, Pkg.BinaryPlatforms, WebCacheUtilities, SHA, Lazy
+using HTTP, JSON, Pkg.BinaryPlatforms, WebCacheUtilities, SHA, Lazy, Tar
 import Pkg.BinaryPlatforms: triplet, arch
+import Pkg.PlatformEngines: exe7z
 
 "Wrapper types to define three jlext methods for portable, tarball and installer Windows"
 struct WindowsPortable
@@ -109,6 +110,11 @@ julia_platforms = [
     FreeBSD(:x86_64),
 ]
 
+const tarball_git_tree_sha1_skiplist = [
+    # Corrupt gzip stream: `7z` reports a CRC failure for the embedded tarball.
+    "https://julialang-s3.julialang.org/bin/linux/x86/0.7/julia-0.7.0-alpha-linux-i686.tar.gz",
+]
+
 function vnum_maybe(x::AbstractString)
     try
         return VersionNumber(x)
@@ -122,6 +128,10 @@ function is_stable(v::VersionNumber)
     return v.prerelease == () && v.build == ()
 end
 
+function tarball_git_tree_sha1(tarball_path::AbstractString)
+    return open(Tar.tree_hash, `$(exe7z()) x $tarball_path -so`)
+end
+
 # Get list of tags from the Julia repo
 function get_tags()
     @info("Probing for tag list...")
@@ -160,6 +170,22 @@ function main(out_path)
             number_urls_success += 1
             println(stdout, " ✓")
 
+            if endswith(filename, ".dmg")
+                kind = "archive"
+                extension = "dmg"
+            elseif endswith(filename, ".exe")
+                kind = "installer"
+                extension = "exe"
+            elseif endswith(filename, ".tar.gz")
+                kind = "archive"
+                extension = "tar.gz"
+            elseif endswith(filename, ".zip")
+                kind = "archive"
+                extension = "zip"
+            else
+                error("Unsupported file extension in filename: $(filename)")
+            end
+
             tarball_hash_path = hit_file_cache("$(filename).sha256") do tarball_hash_path
                 open(filepath, "r") do io
                     open(tarball_hash_path, "w") do hash_io
@@ -169,6 +195,17 @@ function main(out_path)
             end
             tarball_hash = String(read(tarball_hash_path))
 
+            tarball_git_tree_hash = if extension == "tar.gz" && !(url in tarball_git_tree_sha1_skiplist)
+                tarball_git_tree_hash_path = hit_file_cache("$(filename).git-tree-sha1") do tree_hash_path
+                    open(tree_hash_path, "w") do hash_io
+                        write(hash_io, tarball_git_tree_sha1(filepath))
+                    end
+                end
+                String(read(tarball_git_tree_hash_path))
+            else
+                nothing
+            end
+
             # Initialize overall version key, if needed
             if !haskey(meta, version)
                 meta[version] = Dict(
@@ -196,21 +233,6 @@ function main(out_path)
             end
 
             # Build up metadata about this file
-            if endswith(filename, ".dmg")
-                kind = "archive"
-                extension = "dmg"
-            elseif endswith(filename, ".exe")
-                kind = "installer"
-                extension = "exe"
-            elseif endswith(filename, ".tar.gz")
-                kind = "archive"
-                extension = "tar.gz"
-            elseif endswith(filename, ".zip")
-                kind = "archive"
-                extension = "zip"
-            else
-                error("Unsupported file extension in filename: $(filename)")
-            end
             file_dict = Dict(
                 "triplet" => triplet(platform),
                 "os" => meta_os(platform),
@@ -222,6 +244,9 @@ function main(out_path)
                 "extension" => extension,
                 "url" => url,
             )
+            if tarball_git_tree_hash !== nothing
+                file_dict["git-tree-sha1"] = tarball_git_tree_hash
+            end
             # Add in `.asc` signature content, if applicable
             if asc_signature !== nothing
                 file_dict["asc"] = asc_signature

test/runtests.jl

@@ -1,6 +1,8 @@
-using Pkg.BinaryPlatforms, JSON
+using Pkg.BinaryPlatforms, JSON, Tar
 using VersionsJSONUtil
 import VersionsJSONUtil: WindowsPortable, WindowsTarball, MacOSTarball
+import VersionsJSONUtil: tarball_git_tree_sha1, tarball_git_tree_sha1_skiplist
+import Pkg.PlatformEngines: exe7z
 using Test
 
 const download_urls = Dict(
@@ -32,4 +34,33 @@ const download_urls = Dict(
             @test VersionsJSONUtil.download_url(v, p) == url
         end
     end
+
+    @testset "Tarball git-tree-sha1" begin
+        mktempdir() do tempdir
+            srcdir = joinpath(tempdir, "src")
+            mkpath(joinpath(srcdir, "bin"))
+            write(joinpath(srcdir, "README.txt"), "VersionsJSONUtil test fixture\n")
+            write(joinpath(srcdir, "bin", "julia"), "#!/bin/sh\n")
+
+            tar_path = joinpath(tempdir, "fixture.tar")
+            tar_gz_path = joinpath(tempdir, "fixture.tar.gz")
+            Tar.create(srcdir, tar_path)
+            run(`$(exe7z()) a -tgzip $tar_gz_path $tar_path`)
+
+            @test tarball_git_tree_sha1(tar_gz_path) == Tar.tree_hash(tar_path)
+        end
+    end
+
+    @testset "Tarball git-tree-sha1 skiplist" begin
+        @test "https://julialang-s3.julialang.org/bin/linux/x86/0.7/julia-0.7.0-alpha-linux-i686.tar.gz" in tarball_git_tree_sha1_skiplist
+    end
+
+    @testset "Schema tarball requirements" begin
+        schema = JSON.parsefile(joinpath(dirname(@__DIR__), "schema.json"))
+        file_properties = schema["definitions"]["File"]["properties"]
+        file_required = Set(schema["definitions"]["File"]["required"])
+
+        @test haskey(file_properties, "git-tree-sha1")
+        @test !("git-tree-sha1" in file_required)
+    end
 end