From 8a4370b0c6610604f426492025754f4501284537 Mon Sep 17 00:00:00 2001 From: Stefan Krawczyk Date: Sat, 11 Oct 2025 10:44:42 -0700 Subject: [PATCH 1/8] Adds script to help promote RC artifacts This should only move apache-hamilton* artifacts. --- scripts/promote_rc.sh | 107 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 scripts/promote_rc.sh diff --git a/scripts/promote_rc.sh b/scripts/promote_rc.sh new file mode 100644 index 000000000..fac10592a --- /dev/null +++ b/scripts/promote_rc.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# This script moves a release candidate from the dev repository to the release repository. + +set -e + +DRY_RUN=false +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + --dry-run) + DRY_RUN=true + shift # past argument + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + +if [ "$#" -ne 2 ]; then + echo "Usage: $0 [--dry-run] " + echo "Example: $0 --dry-run 1.2.3 0" + exit 1 +fi + +VERSION=$1 +RC_NUM=$2 +PROJECT_SHORT_NAME="hamilton" + +# Source and destination URLs +SOURCE_URL="https://dist.apache.org/repos/dist/dev/incubator/${PROJECT_SHORT_NAME}/${VERSION}-RC${RC_NUM}" +DEST_URL="https://dist.apache.org/repos/dist/release/incubator/${PROJECT_SHORT_NAME}/${VERSION}" + +if [ "$DRY_RUN" = true ]; then + echo "Performing a dry run. No changes will be made." +else + echo "This script will copy the release candidate from dev to release." + echo "Source: ${SOURCE_URL}" + echo "Destination: ${DEST_URL}" + echo " " + + read -p "Are you sure you want to continue? (y/n) " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]] + then + echo "Aborting." + exit 1 + fi +fi + +# Create the release directory +if [ "$DRY_RUN" = true ]; then + echo "DRY RUN: svn mkdir --parents -m \"Creating directory for Apache ${PROJECT_SHORT_NAME} ${VERSION}\" \"${DEST_URL}\"" +else + svn mkdir --parents -m "Creating directory for Apache ${PROJECT_SHORT_NAME} ${VERSION}" "${DEST_URL}" +fi + +# Get the list of files in the source directory +FILES=$(svn list "${SOURCE_URL}") + +for FILE in $FILES; do + if [[ "$FILE" == apache-hamilton* ]]; then + SOURCE_FILE_URL="${SOURCE_URL}/${FILE}" + DEST_FILE_URL="${DEST_URL}/${DEST_FILE_NAME}" + + if [ "$DRY_RUN" = true ]; then + echo "DRY RUN: svn cp \"${SOURCE_FILE_URL}\" \"${DEST_FILE_URL}\" -m \"Promote Apache ${PROJECT_SHORT_NAME} ${VERSION}: ${DEST_FILE_NAME}\"" + else + echo "Copying ${FILE} to ${DEST_FILE_URL}" + svn cp "${SOURCE_FILE_URL}" "${DEST_FILE_URL}" -m "Promote Apache ${PROJECT_SHORT_NAME} ${VERSION}: ${DEST_FILE_NAME}" + fi + fi +done + + +if [ $? -eq 0 ]; then + if [ "$DRY_RUN" = true ]; then + echo "Dry run complete." + else + echo "Successfully copied release artifacts to: ${DEST_URL}" + echo "The release is now live." + fi +else + echo "Error: Failed to copy release artifacts. Please check the SVN URLs and your credentials." + exit 1 +fi From c78e3a19fd79c7e1828e558bba2e8201f312e9cc Mon Sep 17 00:00:00 2001 From: Stefan Krawczyk Date: Sun, 12 Oct 2025 17:17:11 -0700 Subject: [PATCH 2/8] Updates package to be apache-hamilton This will make it simpler to release things. Since we vote on source releases -- this will make it simpler to do so rather than much post-processing. Note: flit for some reason creates metadata that is ahead of what twine knows about. This then updates the helper to drop worrying about the other artifacts. We'll postprocess them to upload to pypi. --- pyproject.toml | 20 +++++- scripts/apache_release_helper.py | 114 ++++++++++++++++++++++++++----- 2 files changed, 116 insertions(+), 18 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 2e58e897e..d8a1e73e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,11 +20,27 @@ requires = ["flit_core >=3.11,<4"] build-backend = "flit_core.buildapi" [project] -name = "sf-hamilton" +name = "apache-hamilton" version = "1.89.0" # NOTE: keep this in sync with hamilton/version.py # TODO: flip back to dynamic once hamilton version is a string. Flit doesn't handle tuples. # dynamic = ["version"] -description = "Hamilton, the micro-framework for creating dataframes." +description = """Apache Hamilton (incubating) is a lightweight Python library for directed acyclic graphs (DAGs) +of transformations. Your DAG is **portable**; it runs anywhere Python runs, whether it's a script, +notebook, Airflow pipeline, FastAPI server, etc. Your DAG is **expressive**; Apache Hamilton has extensive +features to define and modify the execution of a DAG (e.g., data validation, experiment tracking, remote +execution). + +Apache Hamilton (incubating) is an effort undergoing incubation at the Apache +Software Foundation (ASF), sponsored by the Apache Incubator PMC. + +Incubation is required of all newly accepted projects until a further review +indicates that the infrastructure, communications, and decision making process +have stabilized in a manner consistent with other successful ASF projects. + +While incubation status is not necessarily a reflection of the completeness +or stability of the code, it does indicate that the project has yet to be +fully endorsed by the ASF. +""" readme = "README.md" requires-python = ">=3.8.1, <4" license = {text = "Apache-2.0"} diff --git a/scripts/apache_release_helper.py b/scripts/apache_release_helper.py index 9d40f6b31..92b0d2207 100644 --- a/scripts/apache_release_helper.py +++ b/scripts/apache_release_helper.py @@ -22,6 +22,9 @@ import shutil import subprocess import sys +import tarfile +import tempfile +import zipfile from typing import Optional # --- Configuration --- @@ -139,10 +142,96 @@ def sign_artifacts(archive_name: str) -> Optional[list[str]]: return files +def _modify_wheel_for_apache_release(original_wheel: str, new_wheel_path: str): + """Helper to modify the wheel for apache release. + + # Flit somehow builds something incorrectly. + # 1. change PKG-INFO's first line to be `Metadata-Version: 2.4` + # 2. make sure the second line is `Name: apache-hamilton` + # 3. remove the `Import-Name: hamilton` line from PKG-INFO. + + :param original_wheel: Path to the original wheel. + :param new_wheel_path: Path to the new wheel to create. + """ + with tempfile.TemporaryDirectory() as tmpdir: + # Unzip the wheel + with zipfile.ZipFile(original_wheel, "r") as zip_ref: + zip_ref.extractall(tmpdir) + + # Find the .dist-info directory + dist_info_dirs = glob.glob(os.path.join(tmpdir, "*.dist-info")) + if not dist_info_dirs: + raise ValueError(f"Could not find .dist-info directory in {original_wheel}") + dist_info_dir = dist_info_dirs[0] + pkg_info = os.path.join(dist_info_dir, "PKG-INFO") + + _modify_pkg_info_file(pkg_info) + + # Create the new wheel + with zipfile.ZipFile(new_wheel_path, "w", zipfile.ZIP_DEFLATED) as zip_ref: + for root, _, files in os.walk(tmpdir): + for file in files: + zip_ref.write( + os.path.join(root, file), os.path.relpath(os.path.join(root, file), tmpdir) + ) + + +def _modify_pkg_info_file(pkg_info_path: str): + """ + Flit somehow builds something incorrectly. + 1. change PKG-INFO's first line to be `Metadata-Version: 2.4` + 2. make sure the second line is `Name: apache-hamilton` + 3. remove the `Import-Name: hamilton` line from PKG-INFO. + """ + with open(pkg_info_path, "r") as f: + lines = f.readlines() + + new_lines = [] + for i, line in enumerate(lines): + if i == 0: + new_lines.append("Metadata-Version: 2.4\n") + elif i == 1: + new_lines.append("Name: apache-hamilton\n") + elif line.strip() == "Import-Name: hamilton": + continue # Skip this line + else: + new_lines.append(line) + + with open(pkg_info_path, "w") as f: + f.writelines(new_lines) + + +def _modify_tarball_for_apache_release(original_tarball: str, new_tarball_path: str): + """Helper to modify the tarball for apache release. + + # Flit somehow builds something incorrectly. + # 1. change PKG-INFO's first line to be `Metadata-Version: 2.4` + # 2. make sure the second line is `Name: apache-hamilton` + # 3. remove the `Import-Name: hamilton` line from PKG-INFO. + + :param original_tarball: Path to the original tarball. + :param new_tarball_path: Path to the new tarball to create. + """ + with tempfile.TemporaryDirectory() as tmpdir: + # Extract the tarball + with tarfile.open(original_tarball, "r:gz") as tar: + tar.extractall(path=tmpdir) + + # Modify the PKG-INFO file + # The extracted tarball has a single directory inside. + extracted_dir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) + pkg_info_path = os.path.join(extracted_dir, "PKG-INFO") + + _modify_pkg_info_file(pkg_info_path) + + # Create the new tarball + with tarfile.open(new_tarball_path, "w:gz") as tar: + tar.add(extracted_dir, arcname=os.path.basename(extracted_dir)) + + def create_release_artifacts(version) -> list[str]: """Creates the source tarball, GPG signature, and checksums using `python -m build`.""" - print("Creating release artifacts with 'python -m build'...") - files_to_upload = [] + print("Creating release artifacts with 'flit build'...") # Clean the dist directory before building. if os.path.exists("dist"): shutil.rmtree("dist") @@ -162,8 +251,7 @@ def create_release_artifacts(version) -> list[str]: return None # Find the created tarball in the dist directory. - expected_tar_ball = f"dist/sf_hamilton-{version.lower()}.tar.gz" - files_to_upload.append(expected_tar_ball) + expected_tar_ball = f"dist/apache-hamilton-{version.lower()}.tar.gz" tarball_path = glob.glob(expected_tar_ball) if not tarball_path: @@ -180,29 +268,23 @@ def create_release_artifacts(version) -> list[str]: # copy the tarball to be apache-hamilton-{version.lower()}-incubating.tar.gz new_tar_ball = f"dist/apache-hamilton-{version.lower()}-incubating.tar.gz" - shutil.copy(tarball_path[0], new_tar_ball) + # shutil.copy(tarball_path[0], new_tar_ball) + _modify_tarball_for_apache_release(tarball_path[0], new_tar_ball) archive_name = new_tar_ball print(f"Found source tarball: {archive_name}") - main_signed_files = sign_artifacts(archive_name) - if main_signed_files is None: + new_tar_ball_singed = sign_artifacts(archive_name) + if new_tar_ball_singed is None: raise ValueError("Could not sign the main release artifacts.") - # create sf-hamilton release artifacts - sf_hamilton_signed_files = sign_artifacts(expected_tar_ball) # create wheel release artifacts - expected_wheel = f"dist/sf_hamilton-{version.lower()}-py3-none-any.whl" + expected_wheel = f"dist/apache-hamilton-{version.lower()}-py3-none-any.whl" wheel_path = glob.glob(expected_wheel) - wheel_signed_files = sign_artifacts(wheel_path[0]) # create incubator wheel release artifacts expected_incubator_wheel = f"dist/apache-hamilton-{version.lower()}-incubating-py3-none-any.whl" shutil.copy(wheel_path[0], expected_incubator_wheel) incubator_wheel_signed_files = sign_artifacts(expected_incubator_wheel) files_to_upload = ( [new_tar_ball] - + main_signed_files - + [expected_tar_ball] - + sf_hamilton_signed_files - + [expected_wheel] - + wheel_signed_files + + new_tar_ball_singed + [expected_incubator_wheel] + incubator_wheel_signed_files ) From 092ff943bb485a2da768ecd7d708ed1caacf818c Mon Sep 17 00:00:00 2001 From: Stefan Krawczyk Date: Sun, 12 Oct 2025 17:19:30 -0700 Subject: [PATCH 3/8] Removes some README links that shouldn't be there --- README.md | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/README.md b/README.md index 77d0df752..e571cb557 100644 --- a/README.md +++ b/README.md @@ -15,18 +15,12 @@ Total Monthly Downloads
- - - Apache Hamilton Slack - - -

@@ -255,7 +249,7 @@ Apache Hamilton was started at Stitch Fix before the original creators founded D * [wren.ai](https://wren.ai/) ## 🤝 Code Contributors -[![Contributors](https://contrib.rocks/image?repo=dagworks-inc/hamilton)](https://github.com/apache/hamilton/graphs/contributors) +[![Contributors](https://contrib.rocks/image?repo=apache/hamilton)](https://github.com/apache/hamilton/graphs/contributors) ## 🙌 Special Mentions & 🦟 Bug Hunters From 49d07d6b91d250f5d404bb498786403121167bf4 Mon Sep 17 00:00:00 2001 From: Stefan Krawczyk Date: Sat, 18 Oct 2025 14:02:57 -0700 Subject: [PATCH 4/8] Changes copy to mv We should move files to save on disk space. --- scripts/promote_rc.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/promote_rc.sh b/scripts/promote_rc.sh index fac10592a..96c1f6e84 100644 --- a/scripts/promote_rc.sh +++ b/scripts/promote_rc.sh @@ -85,10 +85,10 @@ for FILE in $FILES; do DEST_FILE_URL="${DEST_URL}/${DEST_FILE_NAME}" if [ "$DRY_RUN" = true ]; then - echo "DRY RUN: svn cp \"${SOURCE_FILE_URL}\" \"${DEST_FILE_URL}\" -m \"Promote Apache ${PROJECT_SHORT_NAME} ${VERSION}: ${DEST_FILE_NAME}\"" + echo "DRY RUN: svn mv \"${SOURCE_FILE_URL}\" \"${DEST_FILE_URL}\" -m \"Promote Apache ${PROJECT_SHORT_NAME} ${VERSION}: ${DEST_FILE_NAME}\"" else - echo "Copying ${FILE} to ${DEST_FILE_URL}" - svn cp "${SOURCE_FILE_URL}" "${DEST_FILE_URL}" -m "Promote Apache ${PROJECT_SHORT_NAME} ${VERSION}: ${DEST_FILE_NAME}" + echo "Moving ${FILE} to ${DEST_FILE_URL}" + svn mv "${SOURCE_FILE_URL}" "${DEST_FILE_URL}" -m "Promote Apache ${PROJECT_SHORT_NAME} ${VERSION}: ${DEST_FILE_NAME}" fi fi done From 11249cc8040e8a50d6df0f272124f031f5065c75 Mon Sep 17 00:00:00 2001 From: Stefan Krawczyk Date: Sat, 18 Oct 2025 14:13:32 -0700 Subject: [PATCH 5/8] Adds downloads to docs We missed this in the initial ASF setup. --- docs/asf/downloads.rst | 57 ++++++++++++++++++++++++++++++++++++++++++ docs/asf/index.rst | 2 ++ docs/index.md | 7 ++++++ 3 files changed, 66 insertions(+) create mode 100644 docs/asf/downloads.rst diff --git a/docs/asf/downloads.rst b/docs/asf/downloads.rst new file mode 100644 index 000000000..124f29db1 --- /dev/null +++ b/docs/asf/downloads.rst @@ -0,0 +1,57 @@ +================ +Downloads +================ + +Official Apache Hamilton releases are available at the Apache Software Foundation distribution site. + +Release Downloads +----------------- + +All releases can be found at: `https://downloads.apache.org/incubator/hamilton/ `_ + +Each release includes: + +- Source distributions (.tar.gz, .zip) +- Checksums (SHA512) +- Digital signatures (.asc) + +Verifying Releases +------------------ + +To verify the integrity of downloaded files, you can: + +1. Verify the checksum matches the published SHA512 sum +2. Verify the GPG signature using the KEYS file available at the downloads site + +Release History +--------------- + +Visit the `downloads directory `_ to see all available releases. + +Installation +------------ + +After downloading and verifying a release, you can install it using pip: + +.. code-block:: bash + + pip install apache-hamilton-.tar.gz + +Or install directly from PyPI: + +.. code-block:: bash + + pip install apache-hamilton + +For more information about installation options, see the :doc:`Get Started guide `. + +Official Releases +----------------- + +The following are the official Apache Hamilton releases: + +- **1.89.0** (2025-10-11) - First Apache Hamilton release + + - `Source (tar.gz) `_ + - `Checksums `_ + - `Signature `_ diff --git a/docs/asf/index.rst b/docs/asf/index.rst index 88e9ef25b..b810e6505 100644 --- a/docs/asf/index.rst +++ b/docs/asf/index.rst @@ -10,6 +10,7 @@ Apache Software Foundation links. :glob: :hidden: + downloads Apache Software Foundation License Events @@ -19,6 +20,7 @@ Apache Software Foundation links. Thanks Code of Conduct +- :doc:`Downloads ` - `Foundation `_ - `License `_ - `Events `_ diff --git a/docs/index.md b/docs/index.md index b88d3b610..64559ce1c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -15,6 +15,13 @@ integrations/index code-comparisons/index ``` +```{toctree} +:hidden: True +:caption: DOWNLOADS + +asf/downloads +``` + ```{toctree} :hidden: True :caption: PDF From fe46aece0727995f6487017c55caadf1e94f43b2 Mon Sep 17 00:00:00 2001 From: Stefan Krawczyk Date: Sun, 26 Oct 2025 23:31:52 -0700 Subject: [PATCH 6/8] Update scripts/apache_release_helper.py --- scripts/apache_release_helper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/apache_release_helper.py b/scripts/apache_release_helper.py index 92b0d2207..4170f464f 100644 --- a/scripts/apache_release_helper.py +++ b/scripts/apache_release_helper.py @@ -268,7 +268,6 @@ def create_release_artifacts(version) -> list[str]: # copy the tarball to be apache-hamilton-{version.lower()}-incubating.tar.gz new_tar_ball = f"dist/apache-hamilton-{version.lower()}-incubating.tar.gz" - # shutil.copy(tarball_path[0], new_tar_ball) _modify_tarball_for_apache_release(tarball_path[0], new_tar_ball) archive_name = new_tar_ball print(f"Found source tarball: {archive_name}") From 58ad3c490e003f989baf5a99b47a6e5bed09d906 Mon Sep 17 00:00:00 2001 From: Stefan Krawczyk Date: Sun, 26 Oct 2025 23:32:30 -0700 Subject: [PATCH 7/8] Update scripts/promote_rc.sh --- scripts/promote_rc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/promote_rc.sh b/scripts/promote_rc.sh index 96c1f6e84..7c9798d2d 100644 --- a/scripts/promote_rc.sh +++ b/scripts/promote_rc.sh @@ -85,7 +85,7 @@ for FILE in $FILES; do DEST_FILE_URL="${DEST_URL}/${DEST_FILE_NAME}" if [ "$DRY_RUN" = true ]; then - echo "DRY RUN: svn mv \"${SOURCE_FILE_URL}\" \"${DEST_FILE_URL}\" -m \"Promote Apache ${PROJECT_SHORT_NAME} ${VERSION}: ${DEST_FILE_NAME}\"" + echo "DRY RUN: svn cp \"${SOURCE_FILE_URL}\" \"${DEST_FILE_URL}\" -m \"Promote Apache ${PROJECT_SHORT_NAME} ${VERSION}: ${DEST_FILE_NAME}\"" else echo "Moving ${FILE} to ${DEST_FILE_URL}" svn mv "${SOURCE_FILE_URL}" "${DEST_FILE_URL}" -m "Promote Apache ${PROJECT_SHORT_NAME} ${VERSION}: ${DEST_FILE_NAME}" From 7b66ce418a2e2ffaaf02de44e73f78bcd4e9327d Mon Sep 17 00:00:00 2001 From: Stefan Krawczyk Date: Sun, 26 Oct 2025 23:33:20 -0700 Subject: [PATCH 8/8] Apply suggestions from code review --- scripts/promote_rc.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/promote_rc.sh b/scripts/promote_rc.sh index 7c9798d2d..fac10592a 100644 --- a/scripts/promote_rc.sh +++ b/scripts/promote_rc.sh @@ -87,8 +87,8 @@ for FILE in $FILES; do if [ "$DRY_RUN" = true ]; then echo "DRY RUN: svn cp \"${SOURCE_FILE_URL}\" \"${DEST_FILE_URL}\" -m \"Promote Apache ${PROJECT_SHORT_NAME} ${VERSION}: ${DEST_FILE_NAME}\"" else - echo "Moving ${FILE} to ${DEST_FILE_URL}" - svn mv "${SOURCE_FILE_URL}" "${DEST_FILE_URL}" -m "Promote Apache ${PROJECT_SHORT_NAME} ${VERSION}: ${DEST_FILE_NAME}" + echo "Copying ${FILE} to ${DEST_FILE_URL}" + svn cp "${SOURCE_FILE_URL}" "${DEST_FILE_URL}" -m "Promote Apache ${PROJECT_SHORT_NAME} ${VERSION}: ${DEST_FILE_NAME}" fi fi done