diff --git a/.github/workflows/hamilton-lsp.yml b/.github/workflows/hamilton-lsp.yml index b1d23c224..88458b028 100644 --- a/.github/workflows/hamilton-lsp.yml +++ b/.github/workflows/hamilton-lsp.yml @@ -23,6 +23,9 @@ jobs: working-directory: dev_tools/language_server steps: - uses: actions/checkout@v3 + - name: Check for missing Apache 2 license headers + run: python3 scripts/check_license_headers.py + working-directory: ${{ github.workspace }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: diff --git a/.github/workflows/hamilton-main.yml b/.github/workflows/hamilton-main.yml index b1039c81c..b71e1e384 100644 --- a/.github/workflows/hamilton-main.yml +++ b/.github/workflows/hamilton-main.yml @@ -66,6 +66,11 @@ jobs: uv run pre-commit install uv run pre-commit run --all-files + - name: Check for missing Apache 2 license headers + if: ${{ runner.os == 'Linux' }} + run: | + python3 scripts/check_license_headers.py + - name: Test hamilton main package run: | uv sync --group test diff --git a/.github/workflows/hamilton-sdk.yml b/.github/workflows/hamilton-sdk.yml index d5f539f41..daba9beca 100644 --- a/.github/workflows/hamilton-sdk.yml +++ b/.github/workflows/hamilton-sdk.yml @@ -23,6 +23,9 @@ jobs: working-directory: ui/sdk steps: - uses: actions/checkout@v3 + - name: Check for missing Apache 2 license headers + run: python3 scripts/check_license_headers.py + working-directory: ${{ github.workspace }} - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: diff --git a/.github/workflows/hamilton-ui-backend.yml b/.github/workflows/hamilton-ui-backend.yml index 0249f68b6..204fc48c3 100644 --- a/.github/workflows/hamilton-ui-backend.yml +++ b/.github/workflows/hamilton-ui-backend.yml @@ -36,6 +36,9 @@ jobs: --health-retries 10 steps: - uses: actions/checkout@v3 + - name: Check for missing Apache 2 license headers + run: | + python3 scripts/check_license_headers.py - name: Set up Python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/hamilton-ui-frontend.yml b/.github/workflows/hamilton-ui-frontend.yml index 019898825..a074dc553 100644 --- a/.github/workflows/hamilton-ui-frontend.yml +++ b/.github/workflows/hamilton-ui-frontend.yml @@ -20,6 +20,9 @@ jobs: node-version: [16.x] steps: - uses: actions/checkout@v3 + - name: Check for missing Apache 2 license headers + run: python3 scripts/check_license_headers.py + working-directory: ${{ github.workspace }} - name: Use Node.js ${{ matrix.node-version }} uses: actions/setup-node@v1 with: diff --git a/docs/conf.py b/docs/conf.py index bcccc0d13..a9facf230 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import re import subprocess diff --git a/docs/data_adapters_extension.py b/docs/data_adapters_extension.py index 1cd36c0f1..4139161e9 100644 --- a/docs/data_adapters_extension.py +++ b/docs/data_adapters_extension.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import dataclasses import importlib import inspect diff --git a/docs/make_testimonials.py b/docs/make_testimonials.py index 8f196d3c1..bee435af6 100644 --- a/docs/make_testimonials.py +++ b/docs/make_testimonials.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + card_template = """
diff --git a/examples/Dockerfile b/examples/Dockerfile index fb384b7b0..3ad92f81a 100644 --- a/examples/Dockerfile +++ b/examples/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM python:3.9.14-slim-buster RUN apt update -y &&\ diff --git a/examples/LLM_Workflows/GraphRAG/README.md b/examples/LLM_Workflows/GraphRAG/README.md index c6b5a196a..cf55c680c 100644 --- a/examples/LLM_Workflows/GraphRAG/README.md +++ b/examples/LLM_Workflows/GraphRAG/README.md @@ -1,3 +1,22 @@ + + # Conversational agent over UFC Knowledge graph ## Introduction diff --git a/examples/LLM_Workflows/GraphRAG/application.py b/examples/LLM_Workflows/GraphRAG/application.py index cd01bca6f..a6ee454d6 100644 --- a/examples/LLM_Workflows/GraphRAG/application.py +++ b/examples/LLM_Workflows/GraphRAG/application.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import json import uuid from typing import Tuple diff --git a/examples/LLM_Workflows/GraphRAG/graph_schema.py b/examples/LLM_Workflows/GraphRAG/graph_schema.py index 6d972c4a7..9948955d3 100644 --- a/examples/LLM_Workflows/GraphRAG/graph_schema.py +++ b/examples/LLM_Workflows/GraphRAG/graph_schema.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Code courtesy of the FalkorDB. """ diff --git a/examples/LLM_Workflows/GraphRAG/hamilton_ingest.py b/examples/LLM_Workflows/GraphRAG/hamilton_ingest.py index 2c7e1fb87..b6e59c472 100644 --- a/examples/LLM_Workflows/GraphRAG/hamilton_ingest.py +++ b/examples/LLM_Workflows/GraphRAG/hamilton_ingest.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import ingest_fighters import ingest_fights from falkordb import FalkorDB diff --git a/examples/LLM_Workflows/GraphRAG/ingest_fighters.py b/examples/LLM_Workflows/GraphRAG/ingest_fighters.py index 389095205..1cae04ae3 100644 --- a/examples/LLM_Workflows/GraphRAG/ingest_fighters.py +++ b/examples/LLM_Workflows/GraphRAG/ingest_fighters.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Hamilton pipeline to load fighter data into FalkorDB. """ diff --git a/examples/LLM_Workflows/GraphRAG/ingest_fights.py b/examples/LLM_Workflows/GraphRAG/ingest_fights.py index b907dfd9b..f39586db7 100644 --- a/examples/LLM_Workflows/GraphRAG/ingest_fights.py +++ b/examples/LLM_Workflows/GraphRAG/ingest_fights.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Hamilton module to ingest fight data into FalkorDB. """ diff --git a/examples/LLM_Workflows/GraphRAG/ingest_notebook.ipynb b/examples/LLM_Workflows/GraphRAG/ingest_notebook.ipynb index 875b13ed6..a3ab94f3b 100644 --- a/examples/LLM_Workflows/GraphRAG/ingest_notebook.ipynb +++ b/examples/LLM_Workflows/GraphRAG/ingest_notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/LLM_Workflows/GraphRAG/notebook.ipynb b/examples/LLM_Workflows/GraphRAG/notebook.ipynb index f8ec7ac91..2bbe3d831 100644 --- a/examples/LLM_Workflows/GraphRAG/notebook.ipynb +++ b/examples/LLM_Workflows/GraphRAG/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/LLM_Workflows/GraphRAG/utils.py b/examples/LLM_Workflows/GraphRAG/utils.py index b1d674489..df5c22d88 100644 --- a/examples/LLM_Workflows/GraphRAG/utils.py +++ b/examples/LLM_Workflows/GraphRAG/utils.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Code courtesy of the FalkorDB. """ diff --git a/examples/LLM_Workflows/NER_Example/README.md b/examples/LLM_Workflows/NER_Example/README.md index 00bc5851a..2fabd3879 100644 --- a/examples/LLM_Workflows/NER_Example/README.md +++ b/examples/LLM_Workflows/NER_Example/README.md @@ -1,3 +1,22 @@ + + # Document processing with Named Entity Recognition (NER) for RAG This example demonstrates how to use a Named Entity Recognition (NER) model to extract entities from text along with embeddings to facilitate querying with more precision. Specifically we'll use the entities here to filter to diff --git a/examples/LLM_Workflows/NER_Example/lancedb_module.py b/examples/LLM_Workflows/NER_Example/lancedb_module.py index 790898419..70384a0de 100644 --- a/examples/LLM_Workflows/NER_Example/lancedb_module.py +++ b/examples/LLM_Workflows/NER_Example/lancedb_module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Union import lancedb diff --git a/examples/LLM_Workflows/NER_Example/ner_extraction.py b/examples/LLM_Workflows/NER_Example/ner_extraction.py index 898d9ee55..b8287254e 100644 --- a/examples/LLM_Workflows/NER_Example/ner_extraction.py +++ b/examples/LLM_Workflows/NER_Example/ner_extraction.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Union import torch diff --git a/examples/LLM_Workflows/NER_Example/notebook.ipynb b/examples/LLM_Workflows/NER_Example/notebook.ipynb index cd020984f..8767122ce 100644 --- a/examples/LLM_Workflows/NER_Example/notebook.ipynb +++ b/examples/LLM_Workflows/NER_Example/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/LLM_Workflows/NER_Example/run.py b/examples/LLM_Workflows/NER_Example/run.py index 56b57c94d..47f856e57 100644 --- a/examples/LLM_Workflows/NER_Example/run.py +++ b/examples/LLM_Workflows/NER_Example/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import argparse import lancedb_module diff --git a/examples/LLM_Workflows/RAG_document_extract_chunk_embed/README.md b/examples/LLM_Workflows/RAG_document_extract_chunk_embed/README.md index 05b189a0d..95e360a0a 100644 --- a/examples/LLM_Workflows/RAG_document_extract_chunk_embed/README.md +++ b/examples/LLM_Workflows/RAG_document_extract_chunk_embed/README.md @@ -1,3 +1,22 @@ + + # RAG Document: Extract, Chunk, Embed Example ![pipeline-steps](pipeline-steps.png) diff --git a/examples/LLM_Workflows/RAG_document_extract_chunk_embed/pipeline.py b/examples/LLM_Workflows/RAG_document_extract_chunk_embed/pipeline.py index 9bf866e9b..879f17372 100644 --- a/examples/LLM_Workflows/RAG_document_extract_chunk_embed/pipeline.py +++ b/examples/LLM_Workflows/RAG_document_extract_chunk_embed/pipeline.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Modules that mirrors the pipeline the code in the notebook creates. """ diff --git a/examples/LLM_Workflows/RAG_document_extract_chunk_embed/simple_pipeline.ipynb b/examples/LLM_Workflows/RAG_document_extract_chunk_embed/simple_pipeline.ipynb index d94b93f34..52725104a 100644 --- a/examples/LLM_Workflows/RAG_document_extract_chunk_embed/simple_pipeline.ipynb +++ b/examples/LLM_Workflows/RAG_document_extract_chunk_embed/simple_pipeline.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/LLM_Workflows/README.md b/examples/LLM_Workflows/README.md index 038041c0e..7928a20f0 100644 --- a/examples/LLM_Workflows/README.md +++ b/examples/LLM_Workflows/README.md @@ -1,3 +1,22 @@ + + Here you'll find a collection of workflows that demonstrate how to use Apache Hamilton in various LLM contexts. diff --git a/examples/LLM_Workflows/image_telephone/README.md b/examples/LLM_Workflows/image_telephone/README.md index bc88d2971..04da50f65 100644 --- a/examples/LLM_Workflows/image_telephone/README.md +++ b/examples/LLM_Workflows/image_telephone/README.md @@ -1,3 +1,22 @@ + + # Image Telephone with Burr See the [streamlit app](https://image-telephone.streamlit.app) for a demonstration diff --git a/examples/LLM_Workflows/image_telephone/adapters.py b/examples/LLM_Workflows/image_telephone/adapters.py index dc9e1f934..5c74cfb8b 100644 --- a/examples/LLM_Workflows/image_telephone/adapters.py +++ b/examples/LLM_Workflows/image_telephone/adapters.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import dataclasses import io import json diff --git a/examples/LLM_Workflows/image_telephone/application.py b/examples/LLM_Workflows/image_telephone/application.py index a670fd48b..9af77e433 100644 --- a/examples/LLM_Workflows/image_telephone/application.py +++ b/examples/LLM_Workflows/image_telephone/application.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This module demonstrates a telephone application using Burr that: diff --git a/examples/LLM_Workflows/image_telephone/notebook.ipynb b/examples/LLM_Workflows/image_telephone/notebook.ipynb index 9c7a0b28b..f28ee09a9 100644 --- a/examples/LLM_Workflows/image_telephone/notebook.ipynb +++ b/examples/LLM_Workflows/image_telephone/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/LLM_Workflows/image_telephone/streamlit.py b/examples/LLM_Workflows/image_telephone/streamlit.py index 5a42844c5..2b65ffed3 100644 --- a/examples/LLM_Workflows/image_telephone/streamlit.py +++ b/examples/LLM_Workflows/image_telephone/streamlit.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import collections import concurrent import json diff --git a/examples/LLM_Workflows/knowledge_retrieval/README.md b/examples/LLM_Workflows/knowledge_retrieval/README.md index 6939307ab..8232e61d3 100644 --- a/examples/LLM_Workflows/knowledge_retrieval/README.md +++ b/examples/LLM_Workflows/knowledge_retrieval/README.md @@ -1,3 +1,22 @@ + + # How to use "OpenAI functions" with a Knowledge Base This example is based on [this notebook](https://github.com/openai/openai-cookbook/blob/main/examples/How_to_call_functions_for_knowledge_retrieval.ipynb) diff --git a/examples/LLM_Workflows/knowledge_retrieval/arxiv_articles.py b/examples/LLM_Workflows/knowledge_retrieval/arxiv_articles.py index 4e7772bb6..8a3cad561 100644 --- a/examples/LLM_Workflows/knowledge_retrieval/arxiv_articles.py +++ b/examples/LLM_Workflows/knowledge_retrieval/arxiv_articles.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os.path from typing import Dict diff --git a/examples/LLM_Workflows/knowledge_retrieval/functions.py b/examples/LLM_Workflows/knowledge_retrieval/functions.py index 26a9f03f3..91e19d375 100644 --- a/examples/LLM_Workflows/knowledge_retrieval/functions.py +++ b/examples/LLM_Workflows/knowledge_retrieval/functions.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Module to house functions for an LLM agent to use.""" import logging diff --git a/examples/LLM_Workflows/knowledge_retrieval/state.py b/examples/LLM_Workflows/knowledge_retrieval/state.py index ad177d196..881d61ba1 100644 --- a/examples/LLM_Workflows/knowledge_retrieval/state.py +++ b/examples/LLM_Workflows/knowledge_retrieval/state.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Module that contains code to house state for an agent. The dialog right now is hardcoded at the bottom of this file. diff --git a/examples/LLM_Workflows/knowledge_retrieval/summarize_text.py b/examples/LLM_Workflows/knowledge_retrieval/summarize_text.py index 580756a49..95cc87a81 100644 --- a/examples/LLM_Workflows/knowledge_retrieval/summarize_text.py +++ b/examples/LLM_Workflows/knowledge_retrieval/summarize_text.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import ast import concurrent from typing import Callable, Generator, List diff --git a/examples/LLM_Workflows/langchain_comparison/README.md b/examples/LLM_Workflows/langchain_comparison/README.md index 053a1d078..0188329c1 100644 --- a/examples/LLM_Workflows/langchain_comparison/README.md +++ b/examples/LLM_Workflows/langchain_comparison/README.md @@ -1,3 +1,22 @@ + + # Comparing to LangChain *Rhetorical question*: which code would you rather maintain, change, and update? diff --git a/examples/LLM_Workflows/langchain_comparison/hamilton_anthropic.py b/examples/LLM_Workflows/langchain_comparison/hamilton_anthropic.py index 984adbb0c..dfd8b277a 100644 --- a/examples/LLM_Workflows/langchain_comparison/hamilton_anthropic.py +++ b/examples/LLM_Workflows/langchain_comparison/hamilton_anthropic.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # hamilton_anthropic.py import anthropic import openai diff --git a/examples/LLM_Workflows/langchain_comparison/hamilton_async.py b/examples/LLM_Workflows/langchain_comparison/hamilton_async.py index f399fa86a..551c26455 100644 --- a/examples/LLM_Workflows/langchain_comparison/hamilton_async.py +++ b/examples/LLM_Workflows/langchain_comparison/hamilton_async.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # hamilton_async.py from typing import List diff --git a/examples/LLM_Workflows/langchain_comparison/hamilton_batch.py b/examples/LLM_Workflows/langchain_comparison/hamilton_batch.py index 73b82d64c..8e2161e2a 100644 --- a/examples/LLM_Workflows/langchain_comparison/hamilton_batch.py +++ b/examples/LLM_Workflows/langchain_comparison/hamilton_batch.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # hamilton_batch.py from typing import List diff --git a/examples/LLM_Workflows/langchain_comparison/hamilton_completion.py b/examples/LLM_Workflows/langchain_comparison/hamilton_completion.py index 72109eef0..695f8d983 100644 --- a/examples/LLM_Workflows/langchain_comparison/hamilton_completion.py +++ b/examples/LLM_Workflows/langchain_comparison/hamilton_completion.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # hamilton_completion.py from typing import List diff --git a/examples/LLM_Workflows/langchain_comparison/hamilton_fallbacks.py b/examples/LLM_Workflows/langchain_comparison/hamilton_fallbacks.py index 8ec4dd8c0..8607a191b 100644 --- a/examples/LLM_Workflows/langchain_comparison/hamilton_fallbacks.py +++ b/examples/LLM_Workflows/langchain_comparison/hamilton_fallbacks.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import hamilton_anthropic from hamilton import driver diff --git a/examples/LLM_Workflows/langchain_comparison/hamilton_invoke.py b/examples/LLM_Workflows/langchain_comparison/hamilton_invoke.py index d9e2e731b..0b87e5f4f 100644 --- a/examples/LLM_Workflows/langchain_comparison/hamilton_invoke.py +++ b/examples/LLM_Workflows/langchain_comparison/hamilton_invoke.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # hamilton_invoke.py from typing import List diff --git a/examples/LLM_Workflows/langchain_comparison/hamilton_logging.py b/examples/LLM_Workflows/langchain_comparison/hamilton_logging.py index 4ff50a183..22cf2f039 100644 --- a/examples/LLM_Workflows/langchain_comparison/hamilton_logging.py +++ b/examples/LLM_Workflows/langchain_comparison/hamilton_logging.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # run.py import hamilton_anthropic from hamilton import driver, lifecycle diff --git a/examples/LLM_Workflows/langchain_comparison/hamilton_streamed.py b/examples/LLM_Workflows/langchain_comparison/hamilton_streamed.py index 77669562d..6d998aa6c 100644 --- a/examples/LLM_Workflows/langchain_comparison/hamilton_streamed.py +++ b/examples/LLM_Workflows/langchain_comparison/hamilton_streamed.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # hamilton_streamed.py from typing import Iterator, List diff --git a/examples/LLM_Workflows/langchain_comparison/lcel_anthropic.py b/examples/LLM_Workflows/langchain_comparison/lcel_anthropic.py index 797a2b666..bf10802cb 100644 --- a/examples/LLM_Workflows/langchain_comparison/lcel_anthropic.py +++ b/examples/LLM_Workflows/langchain_comparison/lcel_anthropic.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from langchain_community.chat_models import ChatAnthropic from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import ChatPromptTemplate diff --git a/examples/LLM_Workflows/langchain_comparison/lcel_async.py b/examples/LLM_Workflows/langchain_comparison/lcel_async.py index 8f3ad28b0..7ada1a76e 100644 --- a/examples/LLM_Workflows/langchain_comparison/lcel_async.py +++ b/examples/LLM_Workflows/langchain_comparison/lcel_async.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import RunnablePassthrough diff --git a/examples/LLM_Workflows/langchain_comparison/lcel_batch.py b/examples/LLM_Workflows/langchain_comparison/lcel_batch.py index db0c5ff9a..dd32f6fa2 100644 --- a/examples/LLM_Workflows/langchain_comparison/lcel_batch.py +++ b/examples/LLM_Workflows/langchain_comparison/lcel_batch.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import RunnablePassthrough diff --git a/examples/LLM_Workflows/langchain_comparison/lcel_completion.py b/examples/LLM_Workflows/langchain_comparison/lcel_completion.py index 9ecd1f5ac..f29d48c6a 100644 --- a/examples/LLM_Workflows/langchain_comparison/lcel_completion.py +++ b/examples/LLM_Workflows/langchain_comparison/lcel_completion.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import RunnablePassthrough diff --git a/examples/LLM_Workflows/langchain_comparison/lcel_fallbacks.py b/examples/LLM_Workflows/langchain_comparison/lcel_fallbacks.py index c9750b987..ccebaec29 100644 --- a/examples/LLM_Workflows/langchain_comparison/lcel_fallbacks.py +++ b/examples/LLM_Workflows/langchain_comparison/lcel_fallbacks.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from langchain_community.chat_models import ChatAnthropic, ChatOpenAI from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import ChatPromptTemplate diff --git a/examples/LLM_Workflows/langchain_comparison/lcel_invoke.py b/examples/LLM_Workflows/langchain_comparison/lcel_invoke.py index 33170f139..c8df37bb6 100644 --- a/examples/LLM_Workflows/langchain_comparison/lcel_invoke.py +++ b/examples/LLM_Workflows/langchain_comparison/lcel_invoke.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import RunnablePassthrough diff --git a/examples/LLM_Workflows/langchain_comparison/lcel_logging.py b/examples/LLM_Workflows/langchain_comparison/lcel_logging.py index 0ff834877..743db6b73 100644 --- a/examples/LLM_Workflows/langchain_comparison/lcel_logging.py +++ b/examples/LLM_Workflows/langchain_comparison/lcel_logging.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from langchain_community.chat_models import ChatAnthropic from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import ChatPromptTemplate diff --git a/examples/LLM_Workflows/langchain_comparison/lcel_streamed.py b/examples/LLM_Workflows/langchain_comparison/lcel_streamed.py index cfbed672c..b1c292a97 100644 --- a/examples/LLM_Workflows/langchain_comparison/lcel_streamed.py +++ b/examples/LLM_Workflows/langchain_comparison/lcel_streamed.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import ChatPromptTemplate from langchain_core.runnables import RunnablePassthrough diff --git a/examples/LLM_Workflows/langchain_comparison/vanilla_anthropic.py b/examples/LLM_Workflows/langchain_comparison/vanilla_anthropic.py index 0550ebf57..6f4a16112 100644 --- a/examples/LLM_Workflows/langchain_comparison/vanilla_anthropic.py +++ b/examples/LLM_Workflows/langchain_comparison/vanilla_anthropic.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import anthropic prompt_template = "Tell me a short joke about {topic}" diff --git a/examples/LLM_Workflows/langchain_comparison/vanilla_async.py b/examples/LLM_Workflows/langchain_comparison/vanilla_async.py index 4d6df5aad..6cfc74686 100644 --- a/examples/LLM_Workflows/langchain_comparison/vanilla_async.py +++ b/examples/LLM_Workflows/langchain_comparison/vanilla_async.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import List import openai diff --git a/examples/LLM_Workflows/langchain_comparison/vanilla_batch.py b/examples/LLM_Workflows/langchain_comparison/vanilla_batch.py index e9f275ea1..c87741f5f 100644 --- a/examples/LLM_Workflows/langchain_comparison/vanilla_batch.py +++ b/examples/LLM_Workflows/langchain_comparison/vanilla_batch.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from concurrent.futures import ThreadPoolExecutor from typing import List diff --git a/examples/LLM_Workflows/langchain_comparison/vanilla_completion.py b/examples/LLM_Workflows/langchain_comparison/vanilla_completion.py index 8aa30d6d5..a0b769b6b 100644 --- a/examples/LLM_Workflows/langchain_comparison/vanilla_completion.py +++ b/examples/LLM_Workflows/langchain_comparison/vanilla_completion.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import openai prompt_template = "Tell me a short joke about {topic}" diff --git a/examples/LLM_Workflows/langchain_comparison/vanilla_fallbacks.py b/examples/LLM_Workflows/langchain_comparison/vanilla_fallbacks.py index 2a5a2d91c..7729f5720 100644 --- a/examples/LLM_Workflows/langchain_comparison/vanilla_fallbacks.py +++ b/examples/LLM_Workflows/langchain_comparison/vanilla_fallbacks.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import vanilla_anthropic import vanilla_completion diff --git a/examples/LLM_Workflows/langchain_comparison/vanilla_invoke.py b/examples/LLM_Workflows/langchain_comparison/vanilla_invoke.py index 225835b96..eab368970 100644 --- a/examples/LLM_Workflows/langchain_comparison/vanilla_invoke.py +++ b/examples/LLM_Workflows/langchain_comparison/vanilla_invoke.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import List import openai diff --git a/examples/LLM_Workflows/langchain_comparison/vanilla_logging.py b/examples/LLM_Workflows/langchain_comparison/vanilla_logging.py index d370bd1f4..6b8f1dfdd 100644 --- a/examples/LLM_Workflows/langchain_comparison/vanilla_logging.py +++ b/examples/LLM_Workflows/langchain_comparison/vanilla_logging.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import anthropic prompt_template = "Tell me a short joke about {topic}" diff --git a/examples/LLM_Workflows/langchain_comparison/vanilla_streamed.py b/examples/LLM_Workflows/langchain_comparison/vanilla_streamed.py index c75118d5c..2973acc8f 100644 --- a/examples/LLM_Workflows/langchain_comparison/vanilla_streamed.py +++ b/examples/LLM_Workflows/langchain_comparison/vanilla_streamed.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Iterator, List import openai diff --git a/examples/LLM_Workflows/llm_logging/summarization.py b/examples/LLM_Workflows/llm_logging/summarization.py index 058aea78c..6e9cbab38 100644 --- a/examples/LLM_Workflows/llm_logging/summarization.py +++ b/examples/LLM_Workflows/llm_logging/summarization.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import tempfile from typing import Generator, Union diff --git a/examples/LLM_Workflows/modular_llm_stack/README.md b/examples/LLM_Workflows/modular_llm_stack/README.md index 7dcc005e7..a2aae5e2f 100644 --- a/examples/LLM_Workflows/modular_llm_stack/README.md +++ b/examples/LLM_Workflows/modular_llm_stack/README.md @@ -1,3 +1,22 @@ + + # Flexibly change components of your LLM stack This example shows how to pull data from the HuggingFace datasets hub, create embeddings for text passage using Cohere / OpenAI / SentenceTransformer, and store them in a vector database using LanceDB / Weaviate / Pinecone / Qdrant. diff --git a/examples/LLM_Workflows/modular_llm_stack/data_module.py b/examples/LLM_Workflows/modular_llm_stack/data_module.py index 127bb6578..efc8c1a9b 100644 --- a/examples/LLM_Workflows/modular_llm_stack/data_module.py +++ b/examples/LLM_Workflows/modular_llm_stack/data_module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from datasets import load_dataset diff --git a/examples/LLM_Workflows/modular_llm_stack/embedding_module.py b/examples/LLM_Workflows/modular_llm_stack/embedding_module.py index 4a10f907e..57654d836 100644 --- a/examples/LLM_Workflows/modular_llm_stack/embedding_module.py +++ b/examples/LLM_Workflows/modular_llm_stack/embedding_module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from types import ModuleType import cohere diff --git a/examples/LLM_Workflows/modular_llm_stack/lancedb_module.py b/examples/LLM_Workflows/modular_llm_stack/lancedb_module.py index 09d3913c2..573db8b68 100644 --- a/examples/LLM_Workflows/modular_llm_stack/lancedb_module.py +++ b/examples/LLM_Workflows/modular_llm_stack/lancedb_module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import lancedb import numpy as np import pandas as pd diff --git a/examples/LLM_Workflows/modular_llm_stack/marqo_module.py b/examples/LLM_Workflows/modular_llm_stack/marqo_module.py index a505568dd..e186f7819 100644 --- a/examples/LLM_Workflows/modular_llm_stack/marqo_module.py +++ b/examples/LLM_Workflows/modular_llm_stack/marqo_module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging from typing import Any, Union diff --git a/examples/LLM_Workflows/modular_llm_stack/pinecone_module.py b/examples/LLM_Workflows/modular_llm_stack/pinecone_module.py index 594c701aa..38f4d3bfb 100644 --- a/examples/LLM_Workflows/modular_llm_stack/pinecone_module.py +++ b/examples/LLM_Workflows/modular_llm_stack/pinecone_module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from types import ModuleType import numpy as np diff --git a/examples/LLM_Workflows/modular_llm_stack/qdrant_module.py b/examples/LLM_Workflows/modular_llm_stack/qdrant_module.py index f3bc62cf3..9856cb17e 100644 --- a/examples/LLM_Workflows/modular_llm_stack/qdrant_module.py +++ b/examples/LLM_Workflows/modular_llm_stack/qdrant_module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np from qdrant_client import QdrantClient, models diff --git a/examples/LLM_Workflows/modular_llm_stack/run.py b/examples/LLM_Workflows/modular_llm_stack/run.py index 9a28cc3f1..01af6007e 100644 --- a/examples/LLM_Workflows/modular_llm_stack/run.py +++ b/examples/LLM_Workflows/modular_llm_stack/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import annotations import json diff --git a/examples/LLM_Workflows/modular_llm_stack/weaviate_module.py b/examples/LLM_Workflows/modular_llm_stack/weaviate_module.py index 6def0ec2c..3a723588e 100644 --- a/examples/LLM_Workflows/modular_llm_stack/weaviate_module.py +++ b/examples/LLM_Workflows/modular_llm_stack/weaviate_module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import weaviate diff --git a/examples/LLM_Workflows/observability_openllmetry/README.md b/examples/LLM_Workflows/observability_openllmetry/README.md index d6e2141d8..fff5809bd 100644 --- a/examples/LLM_Workflows/observability_openllmetry/README.md +++ b/examples/LLM_Workflows/observability_openllmetry/README.md @@ -1,3 +1,22 @@ + + # Monitor Apache Hamilton with OpenTelemetry, OpenLLMetry and Traceloop In this simple example, you'll learn how to use the `OpenTelemetryTracer` to emit traces of your Apache Hamilton code using the OpenTelemetry format, in particular LLM applications. diff --git a/examples/LLM_Workflows/observability_openllmetry/notebook.ipynb b/examples/LLM_Workflows/observability_openllmetry/notebook.ipynb index 297b383aa..2fbb29cb0 100644 --- a/examples/LLM_Workflows/observability_openllmetry/notebook.ipynb +++ b/examples/LLM_Workflows/observability_openllmetry/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/LLM_Workflows/observability_openllmetry/run.py b/examples/LLM_Workflows/observability_openllmetry/run.py index 51e5e284e..5b70b9300 100644 --- a/examples/LLM_Workflows/observability_openllmetry/run.py +++ b/examples/LLM_Workflows/observability_openllmetry/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import openai diff --git a/examples/LLM_Workflows/pdf_summarizer/README.md b/examples/LLM_Workflows/pdf_summarizer/README.md index e95f3a754..f760b30ae 100644 --- a/examples/LLM_Workflows/pdf_summarizer/README.md +++ b/examples/LLM_Workflows/pdf_summarizer/README.md @@ -1,3 +1,22 @@ + + # (Yet another) LLM PDF Summarizer 📝 Here's an extensible and production-ready PDF summarizer that you can run anywhere! The frontend uses streamlit, which communicates with a FastAPI backend powered by Apache Hamilton. You give it a PDF file via the browser app and it returns you a text summary using the OpenAI API. If you want, you skip the browser inteface and directly access the `/summarize` endpoint with your document! Everything is containerized using Docker, so you should be able to run it where you please 🏃. diff --git a/examples/LLM_Workflows/pdf_summarizer/backend/Dockerfile b/examples/LLM_Workflows/pdf_summarizer/backend/Dockerfile index a41e52409..894dca775 100644 --- a/examples/LLM_Workflows/pdf_summarizer/backend/Dockerfile +++ b/examples/LLM_Workflows/pdf_summarizer/backend/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM python:3.10-slim-bullseye WORKDIR /app diff --git a/examples/LLM_Workflows/pdf_summarizer/backend/parallel_summarization.py b/examples/LLM_Workflows/pdf_summarizer/backend/parallel_summarization.py index 1b48dcd73..24afdf68c 100644 --- a/examples/LLM_Workflows/pdf_summarizer/backend/parallel_summarization.py +++ b/examples/LLM_Workflows/pdf_summarizer/backend/parallel_summarization.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Module showing the summarization task but using the Parallelizable and Collect features. That is, this code does not embed an executor within the logic of the code. Instead, that is handled by Hamilton. diff --git a/examples/LLM_Workflows/pdf_summarizer/backend/server.py b/examples/LLM_Workflows/pdf_summarizer/backend/server.py index d41d0529e..08ad4bbd9 100644 --- a/examples/LLM_Workflows/pdf_summarizer/backend/server.py +++ b/examples/LLM_Workflows/pdf_summarizer/backend/server.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import base64 import fastapi diff --git a/examples/LLM_Workflows/pdf_summarizer/backend/summarization.py b/examples/LLM_Workflows/pdf_summarizer/backend/summarization.py index 61465fe0f..11df3cc78 100644 --- a/examples/LLM_Workflows/pdf_summarizer/backend/summarization.py +++ b/examples/LLM_Workflows/pdf_summarizer/backend/summarization.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import concurrent import tempfile from typing import Generator, Union diff --git a/examples/LLM_Workflows/pdf_summarizer/frontend/Dockerfile b/examples/LLM_Workflows/pdf_summarizer/frontend/Dockerfile index 089dc33b3..63ad1ee94 100644 --- a/examples/LLM_Workflows/pdf_summarizer/frontend/Dockerfile +++ b/examples/LLM_Workflows/pdf_summarizer/frontend/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM python:3.10-slim-bullseye WORKDIR /app diff --git a/examples/LLM_Workflows/pdf_summarizer/frontend/app.py b/examples/LLM_Workflows/pdf_summarizer/frontend/app.py index 1d7b93a05..093fef413 100644 --- a/examples/LLM_Workflows/pdf_summarizer/frontend/app.py +++ b/examples/LLM_Workflows/pdf_summarizer/frontend/app.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import base64 from pathlib import Path diff --git a/examples/LLM_Workflows/pdf_summarizer/run_on_spark/README.md b/examples/LLM_Workflows/pdf_summarizer/run_on_spark/README.md index 24fdc0884..3a543c128 100644 --- a/examples/LLM_Workflows/pdf_summarizer/run_on_spark/README.md +++ b/examples/LLM_Workflows/pdf_summarizer/run_on_spark/README.md @@ -1,3 +1,22 @@ + + # PDF Summarizer on Spark Here we show how you can run the same Apache Hamilton dataflow, that we defined in the backend diff --git a/examples/LLM_Workflows/pdf_summarizer/run_on_spark/run.ipynb b/examples/LLM_Workflows/pdf_summarizer/run_on_spark/run.ipynb index 9893feb47..770ed369e 100644 --- a/examples/LLM_Workflows/pdf_summarizer/run_on_spark/run.ipynb +++ b/examples/LLM_Workflows/pdf_summarizer/run_on_spark/run.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/LLM_Workflows/pdf_summarizer/run_on_spark/run.py b/examples/LLM_Workflows/pdf_summarizer/run_on_spark/run.py index 4586b521d..5b5036fa0 100644 --- a/examples/LLM_Workflows/pdf_summarizer/run_on_spark/run.py +++ b/examples/LLM_Workflows/pdf_summarizer/run_on_spark/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Spark driver and Hamilton driver code.""" import pandas as pd diff --git a/examples/LLM_Workflows/pdf_summarizer/run_on_spark/run_with_columns.py b/examples/LLM_Workflows/pdf_summarizer/run_on_spark/run_with_columns.py index 32016b2f2..9f27d8233 100644 --- a/examples/LLM_Workflows/pdf_summarizer/run_on_spark/run_with_columns.py +++ b/examples/LLM_Workflows/pdf_summarizer/run_on_spark/run_with_columns.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Spark driver and Hamilton driver code.""" import spark_pdf_pipeline diff --git a/examples/LLM_Workflows/pdf_summarizer/run_on_spark/spark_pdf_pipeline.py b/examples/LLM_Workflows/pdf_summarizer/run_on_spark/spark_pdf_pipeline.py index 31bcb5367..cd3c288ad 100644 --- a/examples/LLM_Workflows/pdf_summarizer/run_on_spark/spark_pdf_pipeline.py +++ b/examples/LLM_Workflows/pdf_summarizer/run_on_spark/spark_pdf_pipeline.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd import pyspark.sql as ps import summarization diff --git a/examples/LLM_Workflows/pdf_summarizer/run_on_spark/summarization.py b/examples/LLM_Workflows/pdf_summarizer/run_on_spark/summarization.py index 7ef05d565..7d8641f78 100644 --- a/examples/LLM_Workflows/pdf_summarizer/run_on_spark/summarization.py +++ b/examples/LLM_Workflows/pdf_summarizer/run_on_spark/summarization.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import concurrent import tempfile from typing import Generator, Union diff --git a/examples/LLM_Workflows/retrieval_augmented_generation/README.md b/examples/LLM_Workflows/retrieval_augmented_generation/README.md index 78d739ba2..1c066a4f4 100644 --- a/examples/LLM_Workflows/retrieval_augmented_generation/README.md +++ b/examples/LLM_Workflows/retrieval_augmented_generation/README.md @@ -1,3 +1,22 @@ + + # Retrieval Augmented Generation app with Apache Hamilton This application allows you to search arXiv for PDFs or import arbitrary PDF files and search over them using LLMs. For each file, the text is divided in chunks that are embedded with OpenAI and stored in Weaviate. When you query the system, the most relevant chunks are retrieved and a summary answer is generated using OpenAI. diff --git a/examples/LLM_Workflows/retrieval_augmented_generation/backend/Dockerfile b/examples/LLM_Workflows/retrieval_augmented_generation/backend/Dockerfile index 2e0cdaf45..5471c3d04 100644 --- a/examples/LLM_Workflows/retrieval_augmented_generation/backend/Dockerfile +++ b/examples/LLM_Workflows/retrieval_augmented_generation/backend/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM python:3.10-slim-bullseye WORKDIR /app diff --git a/examples/LLM_Workflows/retrieval_augmented_generation/backend/ingestion.py b/examples/LLM_Workflows/retrieval_augmented_generation/backend/ingestion.py index dfda3ab7c..14bfedd1f 100644 --- a/examples/LLM_Workflows/retrieval_augmented_generation/backend/ingestion.py +++ b/examples/LLM_Workflows/retrieval_augmented_generation/backend/ingestion.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import base64 import io from pathlib import Path diff --git a/examples/LLM_Workflows/retrieval_augmented_generation/backend/retrieval.py b/examples/LLM_Workflows/retrieval_augmented_generation/backend/retrieval.py index 5935906a0..6a2fb87aa 100644 --- a/examples/LLM_Workflows/retrieval_augmented_generation/backend/retrieval.py +++ b/examples/LLM_Workflows/retrieval_augmented_generation/backend/retrieval.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import openai import weaviate from ingestion import _get_embeddings__openai diff --git a/examples/LLM_Workflows/retrieval_augmented_generation/backend/server.py b/examples/LLM_Workflows/retrieval_augmented_generation/backend/server.py index 5d1690fb9..437a57486 100644 --- a/examples/LLM_Workflows/retrieval_augmented_generation/backend/server.py +++ b/examples/LLM_Workflows/retrieval_augmented_generation/backend/server.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import base64 from contextlib import asynccontextmanager from dataclasses import dataclass diff --git a/examples/LLM_Workflows/retrieval_augmented_generation/backend/vector_db.py b/examples/LLM_Workflows/retrieval_augmented_generation/backend/vector_db.py index 41359398b..adc387071 100644 --- a/examples/LLM_Workflows/retrieval_augmented_generation/backend/vector_db.py +++ b/examples/LLM_Workflows/retrieval_augmented_generation/backend/vector_db.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import weaviate diff --git a/examples/LLM_Workflows/retrieval_augmented_generation/build_app.sh b/examples/LLM_Workflows/retrieval_augmented_generation/build_app.sh index d499da8bd..18f473f7f 100755 --- a/examples/LLM_Workflows/retrieval_augmented_generation/build_app.sh +++ b/examples/LLM_Workflows/retrieval_augmented_generation/build_app.sh @@ -1,4 +1,21 @@ #!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # attribute variables target_dir=$1 api_key=$2 diff --git a/examples/LLM_Workflows/retrieval_augmented_generation/frontend/Dockerfile b/examples/LLM_Workflows/retrieval_augmented_generation/frontend/Dockerfile index 500827495..0e1bf7010 100644 --- a/examples/LLM_Workflows/retrieval_augmented_generation/frontend/Dockerfile +++ b/examples/LLM_Workflows/retrieval_augmented_generation/frontend/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM python:3.10-slim-bullseye WORKDIR /app diff --git a/examples/LLM_Workflows/retrieval_augmented_generation/frontend/Information.py b/examples/LLM_Workflows/retrieval_augmented_generation/frontend/Information.py index 83b31112d..9b8035a97 100644 --- a/examples/LLM_Workflows/retrieval_augmented_generation/frontend/Information.py +++ b/examples/LLM_Workflows/retrieval_augmented_generation/frontend/Information.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import client import streamlit as st diff --git a/examples/LLM_Workflows/retrieval_augmented_generation/frontend/client.py b/examples/LLM_Workflows/retrieval_augmented_generation/frontend/client.py index 04403cbed..afe178938 100644 --- a/examples/LLM_Workflows/retrieval_augmented_generation/frontend/client.py +++ b/examples/LLM_Workflows/retrieval_augmented_generation/frontend/client.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import requests from streamlit.runtime.uploaded_file_manager import UploadedFile diff --git a/examples/LLM_Workflows/retrieval_augmented_generation/frontend/pages/1_Ingestion.py b/examples/LLM_Workflows/retrieval_augmented_generation/frontend/pages/1_Ingestion.py index 74836dc9c..8dcb3ff33 100644 --- a/examples/LLM_Workflows/retrieval_augmented_generation/frontend/pages/1_Ingestion.py +++ b/examples/LLM_Workflows/retrieval_augmented_generation/frontend/pages/1_Ingestion.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import arxiv import client import streamlit as st diff --git a/examples/LLM_Workflows/retrieval_augmented_generation/frontend/pages/2_Retrieval.py b/examples/LLM_Workflows/retrieval_augmented_generation/frontend/pages/2_Retrieval.py index 92bb92d05..61cad2777 100644 --- a/examples/LLM_Workflows/retrieval_augmented_generation/frontend/pages/2_Retrieval.py +++ b/examples/LLM_Workflows/retrieval_augmented_generation/frontend/pages/2_Retrieval.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import client import streamlit as st diff --git a/examples/LLM_Workflows/scraping_and_chunking/README.md b/examples/LLM_Workflows/scraping_and_chunking/README.md index a0b2f6c28..c1de57b93 100644 --- a/examples/LLM_Workflows/scraping_and_chunking/README.md +++ b/examples/LLM_Workflows/scraping_and_chunking/README.md @@ -1,3 +1,22 @@ + + # Scraping and Chunking Scraping and chunking are an important part of any RAG dataflow. Typically they're the start of your "backend" operations to populate for example your vector database. diff --git a/examples/LLM_Workflows/scraping_and_chunking/doc_pipeline.py b/examples/LLM_Workflows/scraping_and_chunking/doc_pipeline.py index c1a383eca..cb616a11f 100644 --- a/examples/LLM_Workflows/scraping_and_chunking/doc_pipeline.py +++ b/examples/LLM_Workflows/scraping_and_chunking/doc_pipeline.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Things this module does. diff --git a/examples/LLM_Workflows/scraping_and_chunking/notebook.ipynb b/examples/LLM_Workflows/scraping_and_chunking/notebook.ipynb index 4b8454e33..dce2fd8cb 100644 --- a/examples/LLM_Workflows/scraping_and_chunking/notebook.ipynb +++ b/examples/LLM_Workflows/scraping_and_chunking/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/LLM_Workflows/scraping_and_chunking/run.py b/examples/LLM_Workflows/scraping_and_chunking/run.py index 256dd66a0..5ca9008b7 100644 --- a/examples/LLM_Workflows/scraping_and_chunking/run.py +++ b/examples/LLM_Workflows/scraping_and_chunking/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ A basic script to run the pipeline defined in `doc_pipeline.py`. diff --git a/examples/LLM_Workflows/scraping_and_chunking/run_dask.py b/examples/LLM_Workflows/scraping_and_chunking/run_dask.py index e8fd2ecf6..3e4346151 100644 --- a/examples/LLM_Workflows/scraping_and_chunking/run_dask.py +++ b/examples/LLM_Workflows/scraping_and_chunking/run_dask.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Shows how to run document chunking using dask. """ diff --git a/examples/LLM_Workflows/scraping_and_chunking/run_ray.py b/examples/LLM_Workflows/scraping_and_chunking/run_ray.py index a85cfc33b..7a6a4cca5 100644 --- a/examples/LLM_Workflows/scraping_and_chunking/run_ray.py +++ b/examples/LLM_Workflows/scraping_and_chunking/run_ray.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Shows how to run document chunking using ray. """ diff --git a/examples/LLM_Workflows/scraping_and_chunking/spark/README.md b/examples/LLM_Workflows/scraping_and_chunking/spark/README.md index e67f0142c..78af824d8 100644 --- a/examples/LLM_Workflows/scraping_and_chunking/spark/README.md +++ b/examples/LLM_Workflows/scraping_and_chunking/spark/README.md @@ -1,3 +1,22 @@ + + # PySpark version of the scraping and chunking example Here we show how you can integrate the Apache Hamilton dataflow, that we defined previously diff --git a/examples/LLM_Workflows/scraping_and_chunking/spark/doc_pipeline.py b/examples/LLM_Workflows/scraping_and_chunking/spark/doc_pipeline.py index a71648795..452c36129 100644 --- a/examples/LLM_Workflows/scraping_and_chunking/spark/doc_pipeline.py +++ b/examples/LLM_Workflows/scraping_and_chunking/spark/doc_pipeline.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This module is a modified version to enable it to be used in a spark job. diff --git a/examples/LLM_Workflows/scraping_and_chunking/spark/notebook.ipynb b/examples/LLM_Workflows/scraping_and_chunking/spark/notebook.ipynb index 2b52742d5..afb4b7605 100644 --- a/examples/LLM_Workflows/scraping_and_chunking/spark/notebook.ipynb +++ b/examples/LLM_Workflows/scraping_and_chunking/spark/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/LLM_Workflows/scraping_and_chunking/spark/spark_pipeline.py b/examples/LLM_Workflows/scraping_and_chunking/spark/spark_pipeline.py index b060c42a3..e7f1fac5a 100644 --- a/examples/LLM_Workflows/scraping_and_chunking/spark/spark_pipeline.py +++ b/examples/LLM_Workflows/scraping_and_chunking/spark/spark_pipeline.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This module creates a spark job that uses the sitemap.xml file to create a dataframe with a URL column, diff --git a/examples/README.md b/examples/README.md index 0fba99e65..a5771283c 100644 --- a/examples/README.md +++ b/examples/README.md @@ -1,3 +1,22 @@ + + # Examples Here you'll find various examples, some relatively simple, to a few that are more complex. As you can see diff --git a/examples/airflow/Dockerfile b/examples/airflow/Dockerfile index e7e826366..6f00a7454 100644 --- a/examples/airflow/Dockerfile +++ b/examples/airflow/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM apache/airflow:latest-python3.10 ADD requirements.txt . RUN pip install -r requirements.txt diff --git a/examples/airflow/README.md b/examples/airflow/README.md index 5b04e3f1d..aa41d4386 100644 --- a/examples/airflow/README.md +++ b/examples/airflow/README.md @@ -1,3 +1,22 @@ + + # Apache Hamilton and Airflow In this example, we're going to show how to run Apache Hamilton within a Airflow task. Both tools are used to author DAGs with Python code, but they operate on different levels: diff --git a/examples/airflow/dags/hamilton/absenteeism_prediction_dag.py b/examples/airflow/dags/hamilton/absenteeism_prediction_dag.py index 74cc62166..5173754b2 100644 --- a/examples/airflow/dags/hamilton/absenteeism_prediction_dag.py +++ b/examples/airflow/dags/hamilton/absenteeism_prediction_dag.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """This is an example project using Hamilton with Apache Airflow For the purpose of this example, we will read and write data from the Airflow diff --git a/examples/airflow/dags/hamilton/hamilton_how_to_dag.py b/examples/airflow/dags/hamilton/hamilton_how_to_dag.py index cdbe3599b..b04935e9c 100644 --- a/examples/airflow/dags/hamilton/hamilton_how_to_dag.py +++ b/examples/airflow/dags/hamilton/hamilton_how_to_dag.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """This file shows different usage pattern to integrate Hamilton with Apache Airflow For the purpose of this example, we will read and write data from the Airflow diff --git a/examples/airflow/plugins/absenteeism/evaluate_model.py b/examples/airflow/plugins/absenteeism/evaluate_model.py index 6664d82a4..ed148b51d 100644 --- a/examples/airflow/plugins/absenteeism/evaluate_model.py +++ b/examples/airflow/plugins/absenteeism/evaluate_model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import scipy.stats as stats from sklearn.metrics import get_scorer diff --git a/examples/airflow/plugins/absenteeism/prepare_data.py b/examples/airflow/plugins/absenteeism/prepare_data.py index f1646e4f0..d4dd2c0a4 100644 --- a/examples/airflow/plugins/absenteeism/prepare_data.py +++ b/examples/airflow/plugins/absenteeism/prepare_data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import List import numpy as np diff --git a/examples/airflow/plugins/absenteeism/train_model.py b/examples/airflow/plugins/absenteeism/train_model.py index d72eb7172..1b5ebe420 100644 --- a/examples/airflow/plugins/absenteeism/train_model.py +++ b/examples/airflow/plugins/absenteeism/train_model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Dict, List import numpy as np diff --git a/examples/airflow/plugins/function_modules/data_loaders.py b/examples/airflow/plugins/function_modules/data_loaders.py index b7a8ed19a..615347e9a 100644 --- a/examples/airflow/plugins/function_modules/data_loaders.py +++ b/examples/airflow/plugins/function_modules/data_loaders.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This module defines data loading logic that we'd like to apply. diff --git a/examples/airflow/plugins/function_modules/feature_logic.py b/examples/airflow/plugins/function_modules/feature_logic.py index 03462cb4a..679b00adb 100644 --- a/examples/airflow/plugins/function_modules/feature_logic.py +++ b/examples/airflow/plugins/function_modules/feature_logic.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This module defines some feature transformation logic that we'd like to apply. It then in addition, decorates the functions with @check_output, so that at runtime, we can diff --git a/examples/async/README.md b/examples/async/README.md index 14bf4fbee..498b545fe 100644 --- a/examples/async/README.md +++ b/examples/async/README.md @@ -1,3 +1,22 @@ + + # Apache Hamilton + Async This is currently an experimental feature, allowing one to run a hamilton DAG composed (entirely or partially) of async functions. diff --git a/examples/async/async_module.py b/examples/async/async_module.py index ad52c7a1b..1db07defe 100644 --- a/examples/async/async_module.py +++ b/examples/async/async_module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import asyncio import json import logging diff --git a/examples/async/fastapi_example.py b/examples/async/fastapi_example.py index 9f3e23682..205fae9cf 100644 --- a/examples/async/fastapi_example.py +++ b/examples/async/fastapi_example.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging from contextlib import asynccontextmanager diff --git a/examples/aws/glue/README.md b/examples/aws/glue/README.md index 652483950..7e7c6f070 100644 --- a/examples/aws/glue/README.md +++ b/examples/aws/glue/README.md @@ -1,3 +1,22 @@ + + # Deploy Apache Hamilton Functions as an AWS Glue Job [AWS Glue](https://aws.amazon.com/glue/) is a serverless data integration service. This guide demonstrates deploying a "hello-world" [processing job](https://docs.aws.amazon.com/glue/latest/dg/add-job-python.html) using Apache Hamilton functions on AWS Glue. diff --git a/examples/aws/glue/app/hamilton_functions/functions.py b/examples/aws/glue/app/hamilton_functions/functions.py index d1644b34d..7468c24fc 100644 --- a/examples/aws/glue/app/hamilton_functions/functions.py +++ b/examples/aws/glue/app/hamilton_functions/functions.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from hamilton.function_modifiers import extract_columns diff --git a/examples/aws/glue/app/setup.py b/examples/aws/glue/app/setup.py index 839975b92..f1fe543ba 100644 --- a/examples/aws/glue/app/setup.py +++ b/examples/aws/glue/app/setup.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from setuptools import setup setup(name="hamilton_functions", version="0.1", packages=["hamilton_functions"]) diff --git a/examples/aws/glue/processing.py b/examples/aws/glue/processing.py index 5b40d5388..06bebe2c3 100644 --- a/examples/aws/glue/processing.py +++ b/examples/aws/glue/processing.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import sys import pandas as pd diff --git a/examples/aws/lambda/Dockerfile b/examples/aws/lambda/Dockerfile index 480f64e38..0e05b8cef 100644 --- a/examples/aws/lambda/Dockerfile +++ b/examples/aws/lambda/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM public.ecr.aws/lambda/python:3.11 COPY requirements.txt ./ diff --git a/examples/aws/lambda/README.md b/examples/aws/lambda/README.md index 3f929cc4b..86a0cf67c 100644 --- a/examples/aws/lambda/README.md +++ b/examples/aws/lambda/README.md @@ -1,3 +1,22 @@ + + # Deploy Apache Hamilton in AWS Lambda [AWS Lambda](https://aws.amazon.com/lambda/) - serverless computation service in AWS. diff --git a/examples/aws/lambda/app/functions.py b/examples/aws/lambda/app/functions.py index d1644b34d..7468c24fc 100644 --- a/examples/aws/lambda/app/functions.py +++ b/examples/aws/lambda/app/functions.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from hamilton.function_modifiers import extract_columns diff --git a/examples/aws/lambda/app/lambda_handler.py b/examples/aws/lambda/app/lambda_handler.py index 225cab51c..48b0bfd05 100644 --- a/examples/aws/lambda/app/lambda_handler.py +++ b/examples/aws/lambda/app/lambda_handler.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from hamilton import driver diff --git a/examples/aws/sagemaker/README.md b/examples/aws/sagemaker/README.md index e7c8f3b9b..da5debccb 100644 --- a/examples/aws/sagemaker/README.md +++ b/examples/aws/sagemaker/README.md @@ -1,3 +1,22 @@ + + # Run Apache Hamilton Functions as an AWS SageMaker Processing Job [AWS SageMaker](https://aws.amazon.com/sagemaker/) is a comprehensive platform that facilitates the creation, training, and deployment of machine learning (ML) models. This guide demonstrates deploying a "hello-world" [processing job](https://docs.aws.amazon.com/sagemaker/latest/dg/processing-job.html) using Apache Hamilton functions on SageMaker. diff --git a/examples/aws/sagemaker/app/functions.py b/examples/aws/sagemaker/app/functions.py index d1644b34d..7468c24fc 100644 --- a/examples/aws/sagemaker/app/functions.py +++ b/examples/aws/sagemaker/app/functions.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from hamilton.function_modifiers import extract_columns diff --git a/examples/aws/sagemaker/container/Dockerfile b/examples/aws/sagemaker/container/Dockerfile index 676303e3d..e4618a2a9 100644 --- a/examples/aws/sagemaker/container/Dockerfile +++ b/examples/aws/sagemaker/container/Dockerfile @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + FROM python:3.11-slim-buster RUN apt-get update && apt-get install -y graphviz diff --git a/examples/aws/sagemaker/notebook.ipynb b/examples/aws/sagemaker/notebook.ipynb index a4692b523..69a00a96c 100644 --- a/examples/aws/sagemaker/notebook.ipynb +++ b/examples/aws/sagemaker/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/aws/sagemaker/processing.py b/examples/aws/sagemaker/processing.py index d6407ae13..7db29eba9 100644 --- a/examples/aws/sagemaker/processing.py +++ b/examples/aws/sagemaker/processing.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from app import functions diff --git a/examples/caching/README.md b/examples/caching/README.md index ac4732a33..5438345e2 100644 --- a/examples/caching/README.md +++ b/examples/caching/README.md @@ -1,3 +1,22 @@ + + # Caching This directory contains tutorial notebooks for the Apache Hamilton caching feature. diff --git a/examples/caching/in_memory_tutorial.ipynb b/examples/caching/in_memory_tutorial.ipynb index 4b4f8ddb4..f7bc9aa2d 100644 --- a/examples/caching/in_memory_tutorial.ipynb +++ b/examples/caching/in_memory_tutorial.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/caching/materializer_tutorial.ipynb b/examples/caching/materializer_tutorial.ipynb index 490075013..81dc2d597 100644 --- a/examples/caching/materializer_tutorial.ipynb +++ b/examples/caching/materializer_tutorial.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/caching/tutorial.ipynb b/examples/caching/tutorial.ipynb index c4d411663..a74621291 100644 --- a/examples/caching/tutorial.ipynb +++ b/examples/caching/tutorial.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/caching_nodes/README.md b/examples/caching_nodes/README.md index e97f5997f..66a07019e 100644 --- a/examples/caching_nodes/README.md +++ b/examples/caching_nodes/README.md @@ -1,3 +1,22 @@ + + # Caching There are four general approaches to caching in Apache Hamilton. diff --git a/examples/caching_nodes/caching.ipynb b/examples/caching_nodes/caching.ipynb index a5d96087d..79f243896 100644 --- a/examples/caching_nodes/caching.ipynb +++ b/examples/caching_nodes/caching.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/caching_nodes/caching_graph_adapter/README.md b/examples/caching_nodes/caching_graph_adapter/README.md index bda41bf6a..aa0c08699 100644 --- a/examples/caching_nodes/caching_graph_adapter/README.md +++ b/examples/caching_nodes/caching_graph_adapter/README.md @@ -1,3 +1,22 @@ + + # Caching Graph Adapter You can use `CachingGraphAdapter` to cache certain nodes. diff --git a/examples/caching_nodes/caching_graph_adapter/business_logic.py b/examples/caching_nodes/caching_graph_adapter/business_logic.py index da3bfb4ae..1cc024bd8 100644 --- a/examples/caching_nodes/caching_graph_adapter/business_logic.py +++ b/examples/caching_nodes/caching_graph_adapter/business_logic.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd """ diff --git a/examples/caching_nodes/caching_graph_adapter/caching_nodes.ipynb b/examples/caching_nodes/caching_graph_adapter/caching_nodes.ipynb index 848194842..bba5303d4 100644 --- a/examples/caching_nodes/caching_graph_adapter/caching_nodes.ipynb +++ b/examples/caching_nodes/caching_graph_adapter/caching_nodes.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/caching_nodes/caching_graph_adapter/data_loaders.py b/examples/caching_nodes/caching_graph_adapter/data_loaders.py index f9fee5842..856c16780 100644 --- a/examples/caching_nodes/caching_graph_adapter/data_loaders.py +++ b/examples/caching_nodes/caching_graph_adapter/data_loaders.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from hamilton.function_modifiers import tag diff --git a/examples/caching_nodes/caching_graph_adapter/run.py b/examples/caching_nodes/caching_graph_adapter/run.py index fbdb70273..3d11488c4 100644 --- a/examples/caching_nodes/caching_graph_adapter/run.py +++ b/examples/caching_nodes/caching_graph_adapter/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging import pathlib import sys diff --git a/examples/caching_nodes/diskcache_adapter/README.md b/examples/caching_nodes/diskcache_adapter/README.md index a4f3ed3e7..5dc20fb39 100644 --- a/examples/caching_nodes/diskcache_adapter/README.md +++ b/examples/caching_nodes/diskcache_adapter/README.md @@ -1,3 +1,22 @@ + + # DiskCache Adapter This adapter uses [diskcache](https://grantjenks.com/docs/diskcache/tutorial.html) to cache node execution on disk. The cache key is a tuple of the function's `(source code, input a, ..., input n)`. This means, a function will only be executed once for a given set of inputs, diff --git a/examples/caching_nodes/diskcache_adapter/functions.py b/examples/caching_nodes/diskcache_adapter/functions.py index 3d693fcde..09df89944 100644 --- a/examples/caching_nodes/diskcache_adapter/functions.py +++ b/examples/caching_nodes/diskcache_adapter/functions.py @@ -1,3 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + def A(external: int) -> int: return external % 7 + 1 diff --git a/examples/caching_nodes/diskcache_adapter/notebook.ipynb b/examples/caching_nodes/diskcache_adapter/notebook.ipynb index 799bb3b57..6997f2eb8 100644 --- a/examples/caching_nodes/diskcache_adapter/notebook.ipynb +++ b/examples/caching_nodes/diskcache_adapter/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/caching_nodes/diskcache_adapter/run.py b/examples/caching_nodes/diskcache_adapter/run.py index 7e445e51f..cc93562ad 100644 --- a/examples/caching_nodes/diskcache_adapter/run.py +++ b/examples/caching_nodes/diskcache_adapter/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging import functions diff --git a/examples/cli/DOCS.md b/examples/cli/DOCS.md index ab9f015a5..fdeb1794e 100644 --- a/examples/cli/DOCS.md +++ b/examples/cli/DOCS.md @@ -1,3 +1,22 @@ + + # `hamilton` Apache Hamilton CLI diff --git a/examples/cli/README.md b/examples/cli/README.md index 04b6cd2f6..6f20110ef 100644 --- a/examples/cli/README.md +++ b/examples/cli/README.md @@ -1,3 +1,22 @@ + + # Apache Hamilton CLI The Apache Hamilton CLI allows to build `Driver` objects from the command line. diff --git a/examples/cli/config.py b/examples/cli/config.py index 80ec6627f..5cace0fd7 100644 --- a/examples/cli/config.py +++ b/examples/cli/config.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + HAMILTON_CONFIG = dict(config_exists="true") HAMILTON_FINAL_VARS = ["config_when", "customer_summary_table"] diff --git a/examples/cli/example_script.py b/examples/cli/example_script.py index caca239f7..ad23e5d3f 100644 --- a/examples/cli/example_script.py +++ b/examples/cli/example_script.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import json import subprocess diff --git a/examples/cli/module_v1.py b/examples/cli/module_v1.py index 2fc7333fa..df6c3362f 100644 --- a/examples/cli/module_v1.py +++ b/examples/cli/module_v1.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from hamilton.function_modifiers import config, extract_columns diff --git a/examples/cli/watcher.py b/examples/cli/watcher.py index 3600985f8..391827636 100644 --- a/examples/cli/watcher.py +++ b/examples/cli/watcher.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import sys from watchdog.events import FileModifiedEvent diff --git a/examples/contrib/README.md b/examples/contrib/README.md index b0263421f..ad4737190 100644 --- a/examples/contrib/README.md +++ b/examples/contrib/README.md @@ -1,3 +1,22 @@ + + # Import Community Dataflows In this example, we show you how to import and use dataflows from the [Apache Hamilton Dataflow Hub](https://hub.dagworks.io/docs/). diff --git a/examples/contrib/notebook.ipynb b/examples/contrib/notebook.ipynb index 9339e9e7f..9a8d39b2e 100644 --- a/examples/contrib/notebook.ipynb +++ b/examples/contrib/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/contrib/notebooks/dagworks-translate_to_hamilton.ipynb b/examples/contrib/notebooks/dagworks-translate_to_hamilton.ipynb index ed36df07d..fba35be9f 100644 --- a/examples/contrib/notebooks/dagworks-translate_to_hamilton.ipynb +++ b/examples/contrib/notebooks/dagworks-translate_to_hamilton.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/contrib/run.py b/examples/contrib/run.py index e2ec3544c..84e167cf5 100644 --- a/examples/contrib/run.py +++ b/examples/contrib/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from sklearn.datasets import load_breast_cancer from sklearn.model_selection import train_test_split diff --git a/examples/dagster/README.md b/examples/dagster/README.md index e68553aee..67f0bdf37 100644 --- a/examples/dagster/README.md +++ b/examples/dagster/README.md @@ -1,3 +1,22 @@ + + # Dagster + Apache Hamilton This repository compares how to build dataflows with macro orchestrator Dagster and the micro orchestrator Apache Hamilton. diff --git a/examples/dagster/dagster_code/README.md b/examples/dagster/dagster_code/README.md index e56a62daf..ad895a871 100644 --- a/examples/dagster/dagster_code/README.md +++ b/examples/dagster/dagster_code/README.md @@ -1,3 +1,22 @@ + + # Dagster This project is adapted from the official [Dagster tutorial](https://docs.dagster.io/tutorial). diff --git a/examples/dagster/dagster_code/setup.py b/examples/dagster/dagster_code/setup.py index bc5ee38d4..83720b4ad 100644 --- a/examples/dagster/dagster_code/setup.py +++ b/examples/dagster/dagster_code/setup.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from setuptools import setup setup(name="tutorial") diff --git a/examples/dagster/dagster_code/tutorial/assets.py b/examples/dagster/dagster_code/tutorial/assets.py index 728a06c44..9b6a3652a 100644 --- a/examples/dagster/dagster_code/tutorial/assets.py +++ b/examples/dagster/dagster_code/tutorial/assets.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import base64 import json import os diff --git a/examples/dagster/hamilton_code/README.md b/examples/dagster/hamilton_code/README.md index 8e3d04971..40e30785d 100644 --- a/examples/dagster/hamilton_code/README.md +++ b/examples/dagster/hamilton_code/README.md @@ -1,3 +1,22 @@ + + # Apache Hamilton ![](./dataflow.png) diff --git a/examples/dagster/hamilton_code/dataflow.py b/examples/dagster/hamilton_code/dataflow.py index 3b4ec060e..0698c91da 100644 --- a/examples/dagster/hamilton_code/dataflow.py +++ b/examples/dagster/hamilton_code/dataflow.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import matplotlib.pyplot as plt import pandas as pd import requests diff --git a/examples/dagster/hamilton_code/mock_api.py b/examples/dagster/hamilton_code/mock_api.py index 7562d0dab..77c5687e8 100644 --- a/examples/dagster/hamilton_code/mock_api.py +++ b/examples/dagster/hamilton_code/mock_api.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from dataclasses import asdict, dataclass from datetime import datetime, timedelta from random import Random diff --git a/examples/dagster/hamilton_code/notebook.ipynb b/examples/dagster/hamilton_code/notebook.ipynb index e3e1958c3..0f10c40dc 100644 --- a/examples/dagster/hamilton_code/notebook.ipynb +++ b/examples/dagster/hamilton_code/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/dagster/hamilton_code/run.py b/examples/dagster/hamilton_code/run.py index b8b59bb42..2a7adf608 100644 --- a/examples/dagster/hamilton_code/run.py +++ b/examples/dagster/hamilton_code/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import dataflow from mock_api import DataGeneratorResource diff --git a/examples/dask/README.md b/examples/dask/README.md index 13760bb5e..733d19bb6 100644 --- a/examples/dask/README.md +++ b/examples/dask/README.md @@ -1,3 +1,22 @@ + + # Scaling Apache Hamilton on Dask [Dask](https://dask.org) is a general purpose framework that allows for parallel diff --git a/examples/dask/community_demo/README.md b/examples/dask/community_demo/README.md index c15995167..285b353b3 100644 --- a/examples/dask/community_demo/README.md +++ b/examples/dask/community_demo/README.md @@ -1,3 +1,22 @@ + + # Dask Community day demo example This example was demo'ed at the Dask community day for July 2023. diff --git a/examples/dask/community_demo/demo_day_notebook.ipynb b/examples/dask/community_demo/demo_day_notebook.ipynb index 671d7f422..128852cd9 100644 --- a/examples/dask/community_demo/demo_day_notebook.ipynb +++ b/examples/dask/community_demo/demo_day_notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/dask/community_demo/my_functions.py b/examples/dask/community_demo/my_functions.py index c4b2a17df..d14da2fcb 100644 --- a/examples/dask/community_demo/my_functions.py +++ b/examples/dask/community_demo/my_functions.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import time import pandas as pd diff --git a/examples/dask/hello_world/README.md b/examples/dask/hello_world/README.md index 61c584635..83006ba67 100644 --- a/examples/dask/hello_world/README.md +++ b/examples/dask/hello_world/README.md @@ -1,3 +1,22 @@ + + # Apache Hamilton on Dask Here we have a hello world example showing how you can diff --git a/examples/dask/hello_world/data_loaders.py b/examples/dask/hello_world/data_loaders.py index 38c4300d1..742147814 100644 --- a/examples/dask/hello_world/data_loaders.py +++ b/examples/dask/hello_world/data_loaders.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from dask import dataframe diff --git a/examples/dask/hello_world/notebook.ipynb b/examples/dask/hello_world/notebook.ipynb index 7caaf82c9..ce74a2d1f 100644 --- a/examples/dask/hello_world/notebook.ipynb +++ b/examples/dask/hello_world/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/dask/hello_world/run.py b/examples/dask/hello_world/run.py index 35e8fe25f..4a8853230 100644 --- a/examples/dask/hello_world/run.py +++ b/examples/dask/hello_world/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import importlib import logging import sys diff --git a/examples/dask/hello_world/run_with_delayed.py b/examples/dask/hello_world/run_with_delayed.py index a8e62ad17..196d080d4 100644 --- a/examples/dask/hello_world/run_with_delayed.py +++ b/examples/dask/hello_world/run_with_delayed.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import importlib import logging import sys diff --git a/examples/dask/hello_world/run_with_delayed_and_dask_objects.py b/examples/dask/hello_world/run_with_delayed_and_dask_objects.py index ef9fa0480..c3b3bcebc 100644 --- a/examples/dask/hello_world/run_with_delayed_and_dask_objects.py +++ b/examples/dask/hello_world/run_with_delayed_and_dask_objects.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import importlib import logging import sys diff --git a/examples/data_loaders/README.md b/examples/data_loaders/README.md index 9d07b3252..82c21aba8 100644 --- a/examples/data_loaders/README.md +++ b/examples/data_loaders/README.md @@ -1,3 +1,22 @@ + + # Data Loaders Among multiple uses, Apache Hamilton excels at building maintainable, scalable representations of ETLs. diff --git a/examples/data_loaders/data_loaders.ipynb b/examples/data_loaders/data_loaders.ipynb index a905e5226..233f3b116 100644 --- a/examples/data_loaders/data_loaders.ipynb +++ b/examples/data_loaders/data_loaders.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/data_loaders/load_data_csv.py b/examples/data_loaders/load_data_csv.py index b8b52851b..8e4a5d49b 100644 --- a/examples/data_loaders/load_data_csv.py +++ b/examples/data_loaders/load_data_csv.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from hamilton.function_modifiers import load_from, value diff --git a/examples/data_loaders/load_data_duckdb.py b/examples/data_loaders/load_data_duckdb.py index 287255c6a..4fe1544f5 100644 --- a/examples/data_loaders/load_data_duckdb.py +++ b/examples/data_loaders/load_data_duckdb.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import duckdb import pandas as pd diff --git a/examples/data_loaders/load_data_mock.py b/examples/data_loaders/load_data_mock.py index 1c7f3f9c8..5ddcfdfd5 100644 --- a/examples/data_loaders/load_data_mock.py +++ b/examples/data_loaders/load_data_mock.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import pandas as pd diff --git a/examples/data_loaders/prep_data.py b/examples/data_loaders/prep_data.py index 6df203e82..987aa4d67 100644 --- a/examples/data_loaders/prep_data.py +++ b/examples/data_loaders/prep_data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from hamilton.function_modifiers import does, extract_columns, parameterize, source, value diff --git a/examples/data_loaders/run.py b/examples/data_loaders/run.py index 088d71f67..d38b185f5 100644 --- a/examples/data_loaders/run.py +++ b/examples/data_loaders/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import click import load_data_csv import load_data_duckdb diff --git a/examples/data_quality/pandera/README.md b/examples/data_quality/pandera/README.md index f9679b502..025ea933f 100644 --- a/examples/data_quality/pandera/README.md +++ b/examples/data_quality/pandera/README.md @@ -1,3 +1,22 @@ + + # Pandera Data Quality Example Here we show how one can define some transformation logic that uses the new Apache Hamilton data quality feature with [Pandera](https://pandera.readthedocs.io/) In addition, we also show how you can manage execution on top of Ray, Pandas on Spark. diff --git a/examples/data_quality/pandera/data_loaders.py b/examples/data_quality/pandera/data_loaders.py index 152898ce6..52c821ac6 100644 --- a/examples/data_quality/pandera/data_loaders.py +++ b/examples/data_quality/pandera/data_loaders.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This module defines data loading logic that we'd like to apply. diff --git a/examples/data_quality/pandera/feature_logic.py b/examples/data_quality/pandera/feature_logic.py index 026ec7710..4060ecbc1 100644 --- a/examples/data_quality/pandera/feature_logic.py +++ b/examples/data_quality/pandera/feature_logic.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This modules shows how to use @check_output with Pandera (https://pandera.readthedocs.io/). diff --git a/examples/data_quality/pandera/feature_logic_spark.py b/examples/data_quality/pandera/feature_logic_spark.py index 8c818ca8e..aa6557bad 100644 --- a/examples/data_quality/pandera/feature_logic_spark.py +++ b/examples/data_quality/pandera/feature_logic_spark.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ We suggest you start by reading `pandera/feature_logic.py` before looking at this file. diff --git a/examples/data_quality/pandera/run.py b/examples/data_quality/pandera/run.py index 29e16ef70..1e818ff0b 100644 --- a/examples/data_quality/pandera/run.py +++ b/examples/data_quality/pandera/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This script runs a Hamilton DAG whose intent is to create some features for input to a model. The model fitting steps are not represented here, just the feature ingestion and transformation logic. diff --git a/examples/data_quality/pandera/run_dask.py b/examples/data_quality/pandera/run_dask.py index 30e497d43..bb5c2e038 100644 --- a/examples/data_quality/pandera/run_dask.py +++ b/examples/data_quality/pandera/run_dask.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This script runs a Hamilton DAG whose intent is to create some features for input to a model. The way the code is set up, is that you can do two things: diff --git a/examples/data_quality/pandera/run_ray.py b/examples/data_quality/pandera/run_ray.py index b2bc2321a..5e6524e8a 100644 --- a/examples/data_quality/pandera/run_ray.py +++ b/examples/data_quality/pandera/run_ray.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This script runs a Hamilton DAG whose intent is to create some features for input to a model. The way the code is set up, is that you can do one thing: diff --git a/examples/data_quality/pandera/run_spark.py b/examples/data_quality/pandera/run_spark.py index 03ad0afee..cc73fc02c 100644 --- a/examples/data_quality/pandera/run_spark.py +++ b/examples/data_quality/pandera/run_spark.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This script runs a Hamilton DAG whose intent is to create some features for input to a model. The way the code is set up, is that you can do one thing: diff --git a/examples/data_quality/simple/README.md b/examples/data_quality/simple/README.md index 3bc7cadcb..57b71a676 100644 --- a/examples/data_quality/simple/README.md +++ b/examples/data_quality/simple/README.md @@ -1,3 +1,22 @@ + + # Data Quality Example Here we show how one can define some transformation logic that uses the new data quality feature. In addition, we also show how you can manage execution on top of Ray, Pandas on Spark. diff --git a/examples/data_quality/simple/data_loaders.py b/examples/data_quality/simple/data_loaders.py index 5c284b904..bb1eb1120 100644 --- a/examples/data_quality/simple/data_loaders.py +++ b/examples/data_quality/simple/data_loaders.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This module defines data loading logic that we'd like to apply. diff --git a/examples/data_quality/simple/feature_logic.py b/examples/data_quality/simple/feature_logic.py index 60a67fd92..962b7c70e 100644 --- a/examples/data_quality/simple/feature_logic.py +++ b/examples/data_quality/simple/feature_logic.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This module defines some feature transformation logic that we'd like to apply. It then in addition, decorates the functions with @check_output, so that at runtime, we can diff --git a/examples/data_quality/simple/run.py b/examples/data_quality/simple/run.py index 29e16ef70..1e818ff0b 100644 --- a/examples/data_quality/simple/run.py +++ b/examples/data_quality/simple/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This script runs a Hamilton DAG whose intent is to create some features for input to a model. The model fitting steps are not represented here, just the feature ingestion and transformation logic. diff --git a/examples/data_quality/simple/run_dask.py b/examples/data_quality/simple/run_dask.py index d61c3db71..3febec7cf 100644 --- a/examples/data_quality/simple/run_dask.py +++ b/examples/data_quality/simple/run_dask.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This script runs a Hamilton DAG whose intent is to create some features for input to a model. The way the code is set up, is that you can do two things: diff --git a/examples/data_quality/simple/run_ray.py b/examples/data_quality/simple/run_ray.py index fa3300455..7c3739ed6 100644 --- a/examples/data_quality/simple/run_ray.py +++ b/examples/data_quality/simple/run_ray.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This script runs a Hamilton DAG whose intent is to create some features for input to a model. The way the code is set up, is that you can do one thing: diff --git a/examples/data_quality/simple/run_spark.py b/examples/data_quality/simple/run_spark.py index e370265af..5003f5ef9 100644 --- a/examples/data_quality/simple/run_spark.py +++ b/examples/data_quality/simple/run_spark.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This script runs a Hamilton DAG whose intent is to create some features for input to a model. The way the code is set up, is that you can do one thing: diff --git a/examples/dbt/README.md b/examples/dbt/README.md index 18df5b375..a141787be 100644 --- a/examples/dbt/README.md +++ b/examples/dbt/README.md @@ -1,3 +1,22 @@ + + # Apache Hamilton and DBT In this example, we're going to show you how easy it is to run Apache Hamilton inside a dbt task. Making use of DBT's exciting new diff --git a/examples/dbt/models/raw_passengers.sql b/examples/dbt/models/raw_passengers.sql index be7e08a94..0b892221c 100644 --- a/examples/dbt/models/raw_passengers.sql +++ b/examples/dbt/models/raw_passengers.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + /* Basic DBT model to load data from the duckdb table diff --git a/examples/dbt/models/train_and_infer.py b/examples/dbt/models/train_and_infer.py index 216cf313b..45bce6f19 100644 --- a/examples/dbt/models/train_and_infer.py +++ b/examples/dbt/models/train_and_infer.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from python_transforms import data_loader, feature_transforms, model_pipeline diff --git a/examples/dbt/python_transforms/data_loader.py b/examples/dbt/python_transforms/data_loader.py index e0eeba896..dfe8f3d72 100644 --- a/examples/dbt/python_transforms/data_loader.py +++ b/examples/dbt/python_transforms/data_loader.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This module contains our data loading functions. """ diff --git a/examples/dbt/python_transforms/feature_transforms.py b/examples/dbt/python_transforms/feature_transforms.py index 5feb3de41..04b71c708 100644 --- a/examples/dbt/python_transforms/feature_transforms.py +++ b/examples/dbt/python_transforms/feature_transforms.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This is a module that contains our feature transforms. """ diff --git a/examples/dbt/python_transforms/model_pipeline.py b/examples/dbt/python_transforms/model_pipeline.py index 1b30916c7..0c1695345 100644 --- a/examples/dbt/python_transforms/model_pipeline.py +++ b/examples/dbt/python_transforms/model_pipeline.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This is a module that contains our "model fitting and related" transforms. """ diff --git a/examples/decoupling_io/README.md b/examples/decoupling_io/README.md index b3f53d16a..8d42dd30d 100644 --- a/examples/decoupling_io/README.md +++ b/examples/decoupling_io/README.md @@ -1,3 +1,22 @@ + + # Get started This accompanies the blog post on decoupling IO from transformation: https://blog.dagworks.io/p/separate-data-io-from-transformation. diff --git a/examples/decoupling_io/adapters.py b/examples/decoupling_io/adapters.py index b7e6d515f..e26060367 100644 --- a/examples/decoupling_io/adapters.py +++ b/examples/decoupling_io/adapters.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import dataclasses from os import PathLike from typing import Any, Collection, Dict, Optional, Type, Union diff --git a/examples/decoupling_io/components/feature_data.py b/examples/decoupling_io/components/feature_data.py index 788c875f4..fa63368fe 100644 --- a/examples/decoupling_io/components/feature_data.py +++ b/examples/decoupling_io/components/feature_data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This is a module that contains our feature transforms. """ diff --git a/examples/decoupling_io/components/model_evaluation.py b/examples/decoupling_io/components/model_evaluation.py index 3cc1b42ec..9ff2944fa 100644 --- a/examples/decoupling_io/components/model_evaluation.py +++ b/examples/decoupling_io/components/model_evaluation.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import pandas as pd from sklearn import base, metrics diff --git a/examples/decoupling_io/components/model_training.py b/examples/decoupling_io/components/model_training.py index d1e7806f7..44fee2df4 100644 --- a/examples/decoupling_io/components/model_training.py +++ b/examples/decoupling_io/components/model_training.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This is a module that contains our "model fitting and related" transforms. """ diff --git a/examples/decoupling_io/run.py b/examples/decoupling_io/run.py index ac890d9ad..4a9d1e4d8 100644 --- a/examples/decoupling_io/run.py +++ b/examples/decoupling_io/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Since this is not in plugins we need to import it *before* doing anything else import importlib from typing import List, Union diff --git a/examples/dlt/README.md b/examples/dlt/README.md index 094505884..0017c5b3b 100644 --- a/examples/dlt/README.md +++ b/examples/dlt/README.md @@ -1,3 +1,22 @@ + + # dlt This example shows dlt + Apache Hamilton can help you cover the full ELT cycle using portable Python code. It is a companion to this [documentation page](https://hamilton.apache.org/integrations/dlt/). diff --git a/examples/dlt/dlt_plugin.ipynb b/examples/dlt/dlt_plugin.ipynb index f3e34577c..09d215c9c 100644 --- a/examples/dlt/dlt_plugin.ipynb +++ b/examples/dlt/dlt_plugin.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/dlt/notebook.ipynb b/examples/dlt/notebook.ipynb index a399bd088..a784e7cb7 100644 --- a/examples/dlt/notebook.ipynb +++ b/examples/dlt/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/dlt/run.py b/examples/dlt/run.py index 876dc3f30..cdcbc5e2f 100644 --- a/examples/dlt/run.py +++ b/examples/dlt/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import dlt diff --git a/examples/dlt/slack/README.md b/examples/dlt/slack/README.md index 5ddc90454..19e623db0 100644 --- a/examples/dlt/slack/README.md +++ b/examples/dlt/slack/README.md @@ -1,3 +1,22 @@ + + --- title: Slack description: dlt source for Slack API diff --git a/examples/dlt/slack/helpers.py b/examples/dlt/slack/helpers.py index 83078e7f7..519c6a338 100644 --- a/examples/dlt/slack/helpers.py +++ b/examples/dlt/slack/helpers.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Slack source helpers.""" from typing import Any, Generator, Iterable, List, Optional diff --git a/examples/dlt/slack/settings.py b/examples/dlt/slack/settings.py index 34040a9d7..1b6d8e4c6 100644 --- a/examples/dlt/slack/settings.py +++ b/examples/dlt/slack/settings.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Slack source settings and constants""" from dlt.common import pendulum diff --git a/examples/dlt/transform.py b/examples/dlt/transform.py index d8c7a1f29..e3baa851f 100644 --- a/examples/dlt/transform.py +++ b/examples/dlt/transform.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import textwrap import dlt diff --git a/examples/due_date_probabilities/README.md b/examples/due_date_probabilities/README.md index 1c0a1d12a..c7ce2f322 100644 --- a/examples/due_date_probabilities/README.md +++ b/examples/due_date_probabilities/README.md @@ -1,3 +1,22 @@ + + # Modeling Pregnancy Due Dates using Apache Hamilton This is an example of developing and applying a simple statistical model using Apache Hamilton diff --git a/examples/due_date_probabilities/base_dates.py b/examples/due_date_probabilities/base_dates.py index d14fa3b75..85df866ed 100644 --- a/examples/due_date_probabilities/base_dates.py +++ b/examples/due_date_probabilities/base_dates.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import datetime import pandas as pd diff --git a/examples/due_date_probabilities/notebook.ipynb b/examples/due_date_probabilities/notebook.ipynb index 6c4879bc3..a4ed3fa67 100644 --- a/examples/due_date_probabilities/notebook.ipynb +++ b/examples/due_date_probabilities/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/due_date_probabilities/probabilities.py b/examples/due_date_probabilities/probabilities.py index 8f55fb821..832cf8d28 100644 --- a/examples/due_date_probabilities/probabilities.py +++ b/examples/due_date_probabilities/probabilities.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import datetime from typing import Optional diff --git a/examples/due_date_probabilities/probability_estimation.py b/examples/due_date_probabilities/probability_estimation.py index 5c673e5ed..a3878750a 100644 --- a/examples/due_date_probabilities/probability_estimation.py +++ b/examples/due_date_probabilities/probability_estimation.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import List import pandas as pd diff --git a/examples/experiment_management/README.md b/examples/experiment_management/README.md index 355165d15..2e6eea4f2 100644 --- a/examples/experiment_management/README.md +++ b/examples/experiment_management/README.md @@ -1,3 +1,22 @@ + + # Apache Hamilton Experiment Manager Add a hook to your Apache Hamilton Driver to log runs and visualize artifacts and metadata! The server is built using FastAPI + FastUI allowing to easily integrate the server within your app or extend the UI. diff --git a/examples/experiment_management/analysis.py b/examples/experiment_management/analysis.py index d0d1b8bb2..457648836 100644 --- a/examples/experiment_management/analysis.py +++ b/examples/experiment_management/analysis.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import matplotlib.pyplot as plt import numpy as np import pandas as pd diff --git a/examples/experiment_management/notebook.ipynb b/examples/experiment_management/notebook.ipynb index 510f30a08..f4466fbe7 100644 --- a/examples/experiment_management/notebook.ipynb +++ b/examples/experiment_management/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/experiment_management/run.py b/examples/experiment_management/run.py index 37526e34b..70946d6f2 100644 --- a/examples/experiment_management/run.py +++ b/examples/experiment_management/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import analysis from hamilton import base, driver diff --git a/examples/feast/README.md b/examples/feast/README.md index a9ec5cef1..7b4001f7c 100644 --- a/examples/feast/README.md +++ b/examples/feast/README.md @@ -1,3 +1,22 @@ + + # Apache Hamilton + Feast In this example, we're going to show you how Apache Hamilton can help you structure your Feast repository and bring tighter coupling between your feature transformations (code) and feature store (data). diff --git a/examples/feast/default_feature_store/README.md b/examples/feast/default_feature_store/README.md index 1544ff1c7..8317f236c 100644 --- a/examples/feast/default_feature_store/README.md +++ b/examples/feast/default_feature_store/README.md @@ -1,3 +1,22 @@ + + # Context Anything in the directory `/default_feature_store`, including the rest of this `README` was generated with the CLI command `feast init default_feature_store`, which creates a Feast quickstart example. This example can provide useful information to new Feast users and serve as a comparison point to integrations with Apache Hamilton. The directory `/simple_feature_store` is reimplements 1-to-1 the functionalities from this directory. diff --git a/examples/feast/default_feature_store/feature_repo/example_repo.py b/examples/feast/default_feature_store/feature_repo/example_repo.py index 47f6a45c5..204ee6c9f 100644 --- a/examples/feast/default_feature_store/feature_repo/example_repo.py +++ b/examples/feast/default_feature_store/feature_repo/example_repo.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # This is an example feature definition file from datetime import timedelta diff --git a/examples/feast/default_feature_store/feature_repo/test_workflow.py b/examples/feast/default_feature_store/feature_repo/test_workflow.py index bb54735f9..40076dc64 100644 --- a/examples/feast/default_feature_store/feature_repo/test_workflow.py +++ b/examples/feast/default_feature_store/feature_repo/test_workflow.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import subprocess from datetime import datetime diff --git a/examples/feast/integration_feature_store/feature_repo/demo_inputs.py b/examples/feast/integration_feature_store/feature_repo/demo_inputs.py index c13d032a3..9c5da0876 100644 --- a/examples/feast/integration_feature_store/feature_repo/demo_inputs.py +++ b/examples/feast/integration_feature_store/feature_repo/demo_inputs.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """This module defines example inputs to use with Feast feature retrieval patterns""" from datetime import datetime diff --git a/examples/feast/integration_feature_store/feature_repo/feature_transformations.py b/examples/feast/integration_feature_store/feature_repo/feature_transformations.py index da1fc741e..f5ba9c505 100644 --- a/examples/feast/integration_feature_store/feature_repo/feature_transformations.py +++ b/examples/feast/integration_feature_store/feature_repo/feature_transformations.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import pandas as pd diff --git a/examples/feast/integration_feature_store/feature_repo/retrieval.ipynb b/examples/feast/integration_feature_store/feature_repo/retrieval.ipynb index 2f75af7b8..1b702f6b1 100644 --- a/examples/feast/integration_feature_store/feature_repo/retrieval.ipynb +++ b/examples/feast/integration_feature_store/feature_repo/retrieval.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/feast/integration_feature_store/feature_repo/run.py b/examples/feast/integration_feature_store/feature_repo/run.py index 178b446d0..5ccb3f502 100644 --- a/examples/feast/integration_feature_store/feature_repo/run.py +++ b/examples/feast/integration_feature_store/feature_repo/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import store_definitions import store_operations diff --git a/examples/feast/integration_feature_store/feature_repo/store_definitions.py b/examples/feast/integration_feature_store/feature_repo/store_definitions.py index 0b9f8f927..5789b24cd 100644 --- a/examples/feast/integration_feature_store/feature_repo/store_definitions.py +++ b/examples/feast/integration_feature_store/feature_repo/store_definitions.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """This module contains the definitions of Feast objects pushed to the registry It allows you to define your Feast objects dependencies as a Hamilton DAG and create lineage with your feature transforms defined in Hamilton. diff --git a/examples/feast/integration_feature_store/feature_repo/store_operations.py b/examples/feast/integration_feature_store/feature_repo/store_operations.py index b9f78b7e3..1f9ca83ea 100644 --- a/examples/feast/integration_feature_store/feature_repo/store_operations.py +++ b/examples/feast/integration_feature_store/feature_repo/store_operations.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """This module contains the operations to execute on the Feast FeatureStore""" from datetime import datetime diff --git a/examples/feast/simple_feature_store/feature_repo/demo_inputs.py b/examples/feast/simple_feature_store/feature_repo/demo_inputs.py index c13d032a3..9c5da0876 100644 --- a/examples/feast/simple_feature_store/feature_repo/demo_inputs.py +++ b/examples/feast/simple_feature_store/feature_repo/demo_inputs.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """This module defines example inputs to use with Feast feature retrieval patterns""" from datetime import datetime diff --git a/examples/feast/simple_feature_store/feature_repo/run.py b/examples/feast/simple_feature_store/feature_repo/run.py index 95f69cd4f..b4c74ab87 100644 --- a/examples/feast/simple_feature_store/feature_repo/run.py +++ b/examples/feast/simple_feature_store/feature_repo/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import store_definitions import store_operations diff --git a/examples/feast/simple_feature_store/feature_repo/store_definitions.py b/examples/feast/simple_feature_store/feature_repo/store_definitions.py index c3aa66b3b..74bdae40d 100644 --- a/examples/feast/simple_feature_store/feature_repo/store_definitions.py +++ b/examples/feast/simple_feature_store/feature_repo/store_definitions.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """This module contains the definitions of Feast objects pushed to the registry It allows you to define your Feast objects dependencies as a Hamilton DAG """ diff --git a/examples/feast/simple_feature_store/feature_repo/store_operations.py b/examples/feast/simple_feature_store/feature_repo/store_operations.py index 308f5430a..e86f3de00 100644 --- a/examples/feast/simple_feature_store/feature_repo/store_operations.py +++ b/examples/feast/simple_feature_store/feature_repo/store_operations.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """This module contains the operations to execute on the Feast FeatureStore""" from datetime import datetime diff --git a/examples/feature_engineering/README.md b/examples/feature_engineering/README.md index d95042df9..8dfb9202c 100644 --- a/examples/feature_engineering/README.md +++ b/examples/feature_engineering/README.md @@ -1,3 +1,22 @@ + + # Feature Engineering Here you'll find two examples. diff --git a/examples/feature_engineering/feature_engineering_multiple_contexts/README.md b/examples/feature_engineering/feature_engineering_multiple_contexts/README.md index 6e6f8c30a..116b92ad0 100644 --- a/examples/feature_engineering/feature_engineering_multiple_contexts/README.md +++ b/examples/feature_engineering/feature_engineering_multiple_contexts/README.md @@ -1,3 +1,22 @@ + + # Feature Engineering in Multiple Contexts What is feature engineering? It's the process of transforming data for input to a "model". diff --git a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/README.md b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/README.md index 2cb0e3c21..b8666b1be 100644 --- a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/README.md +++ b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/README.md @@ -1,3 +1,22 @@ + + # Scenario 1: the simple case - ETL + Online API Context required to understand this scenario: diff --git a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/etl.py b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/etl.py index 38d31b9cd..0dcd15fc6 100644 --- a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/etl.py +++ b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/etl.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This is part of an offline ETL that you'd likely have. You pull data from a source, and transform it into features, and then save/fit a model with them. diff --git a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/fastapi_server.py b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/fastapi_server.py index 296c906d1..7d518ceee 100644 --- a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/fastapi_server.py +++ b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/fastapi_server.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ " This is a simple example of a FastAPI server that uses Hamilton on the request path to transform the data into features, and then uses a fake model to make diff --git a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/features.py b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/features.py index 03fa20d91..de02b9282 100644 --- a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/features.py +++ b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/features.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Here are some features that are based on the absenteeism data set. They are just supposed to be illustrative of the kind of features one might have. diff --git a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/named_model_feature_sets.py b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/named_model_feature_sets.py index bb1be2b48..a85793fb4 100644 --- a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/named_model_feature_sets.py +++ b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/named_model_feature_sets.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This module is a light weight way to "record" feature sets that are used in a model; just define it in code! diff --git a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/offline_loader.py b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/offline_loader.py index 00a983c3d..a2564aa74 100644 --- a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/offline_loader.py +++ b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_1/offline_loader.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Module that contains logic to load data for the offline ETL process. diff --git a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/README.md b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/README.md index ccc8d20ac..6d3f7c6a1 100644 --- a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/README.md +++ b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/README.md @@ -1,3 +1,22 @@ + + # Scenario 2: the more complex case - request doesn't have the raw data - ETL + Online API Context required to understand this scenario: diff --git a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/etl.py b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/etl.py index 3eabf83f9..1c2a02c8d 100644 --- a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/etl.py +++ b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/etl.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This is part of an offline ETL that you'd likely have. You pull data from a source, and transform it into features, and then save/fit a model with them. diff --git a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/fastapi_server.py b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/fastapi_server.py index 6711cbaae..805c66b6c 100644 --- a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/fastapi_server.py +++ b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/fastapi_server.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ " This is a simple example of a FastAPI server that uses Hamilton on the request path to transform the data into features, and then uses a fake model to make diff --git a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/features.py b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/features.py index a006759c1..1ea8f800f 100644 --- a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/features.py +++ b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/features.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Here are some features that are based on the absenteeism data set. They are just supposed to be illustrative of the kind of features one might have. diff --git a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/named_model_feature_sets.py b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/named_model_feature_sets.py index bb1be2b48..a85793fb4 100644 --- a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/named_model_feature_sets.py +++ b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/named_model_feature_sets.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This module is a light weight way to "record" feature sets that are used in a model; just define it in code! diff --git a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/offline_loader.py b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/offline_loader.py index ac50159b1..a62bff3d2 100644 --- a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/offline_loader.py +++ b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/offline_loader.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Module that contains logic to load data for the offline ETL process. diff --git a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/online_loader.py b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/online_loader.py index 1a389e2f2..7e3f06157 100644 --- a/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/online_loader.py +++ b/examples/feature_engineering/feature_engineering_multiple_contexts/scenario_2/online_loader.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Module that contains logic to load data for the offline ETL process. diff --git a/examples/feature_engineering/write_once_run_everywhere_blog_post/README.md b/examples/feature_engineering/write_once_run_everywhere_blog_post/README.md index a5a246784..8ffcb22b8 100644 --- a/examples/feature_engineering/write_once_run_everywhere_blog_post/README.md +++ b/examples/feature_engineering/write_once_run_everywhere_blog_post/README.md @@ -1,3 +1,22 @@ + + # Overview This code accompanies the ["Write once, run everywhere" blog post](https://blog.dagworks.io/p/feature-engineering-with-hamilton). diff --git a/examples/feature_engineering/write_once_run_everywhere_blog_post/components/aggregations.py b/examples/feature_engineering/write_once_run_everywhere_blog_post/components/aggregations.py index db95847da..5885d3fa4 100644 --- a/examples/feature_engineering/write_once_run_everywhere_blog_post/components/aggregations.py +++ b/examples/feature_engineering/write_once_run_everywhere_blog_post/components/aggregations.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from components import utils diff --git a/examples/feature_engineering/write_once_run_everywhere_blog_post/components/data_loaders.py b/examples/feature_engineering/write_once_run_everywhere_blog_post/components/data_loaders.py index d83b32955..56e1726ae 100644 --- a/examples/feature_engineering/write_once_run_everywhere_blog_post/components/data_loaders.py +++ b/examples/feature_engineering/write_once_run_everywhere_blog_post/components/data_loaders.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from components import utils diff --git a/examples/feature_engineering/write_once_run_everywhere_blog_post/components/features.py b/examples/feature_engineering/write_once_run_everywhere_blog_post/components/features.py index 86155d8f7..fc704f9f9 100644 --- a/examples/feature_engineering/write_once_run_everywhere_blog_post/components/features.py +++ b/examples/feature_engineering/write_once_run_everywhere_blog_post/components/features.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from datetime import datetime import pandas as pd diff --git a/examples/feature_engineering/write_once_run_everywhere_blog_post/components/joins.py b/examples/feature_engineering/write_once_run_everywhere_blog_post/components/joins.py index 49da27af7..85df06b44 100644 --- a/examples/feature_engineering/write_once_run_everywhere_blog_post/components/joins.py +++ b/examples/feature_engineering/write_once_run_everywhere_blog_post/components/joins.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from hamilton.function_modifiers import config diff --git a/examples/feature_engineering/write_once_run_everywhere_blog_post/components/model.py b/examples/feature_engineering/write_once_run_everywhere_blog_post/components/model.py index a94fc41ff..f3592c571 100644 --- a/examples/feature_engineering/write_once_run_everywhere_blog_post/components/model.py +++ b/examples/feature_engineering/write_once_run_everywhere_blog_post/components/model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import random import pandas as pd diff --git a/examples/feature_engineering/write_once_run_everywhere_blog_post/components/utils.py b/examples/feature_engineering/write_once_run_everywhere_blog_post/components/utils.py index c7c2d0c3d..205ff6dfa 100644 --- a/examples/feature_engineering/write_once_run_everywhere_blog_post/components/utils.py +++ b/examples/feature_engineering/write_once_run_everywhere_blog_post/components/utils.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import random from typing import List diff --git a/examples/feature_engineering/write_once_run_everywhere_blog_post/contexts/batch.py b/examples/feature_engineering/write_once_run_everywhere_blog_post/contexts/batch.py index 761cec6cb..5daf9c422 100644 --- a/examples/feature_engineering/write_once_run_everywhere_blog_post/contexts/batch.py +++ b/examples/feature_engineering/write_once_run_everywhere_blog_post/contexts/batch.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import datetime import click diff --git a/examples/feature_engineering/write_once_run_everywhere_blog_post/contexts/online.py b/examples/feature_engineering/write_once_run_everywhere_blog_post/contexts/online.py index f91658c29..53f18e348 100644 --- a/examples/feature_engineering/write_once_run_everywhere_blog_post/contexts/online.py +++ b/examples/feature_engineering/write_once_run_everywhere_blog_post/contexts/online.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from datetime import datetime import click diff --git a/examples/feature_engineering/write_once_run_everywhere_blog_post/contexts/streaming.py b/examples/feature_engineering/write_once_run_everywhere_blog_post/contexts/streaming.py index 20ddacbdf..482df044f 100644 --- a/examples/feature_engineering/write_once_run_everywhere_blog_post/contexts/streaming.py +++ b/examples/feature_engineering/write_once_run_everywhere_blog_post/contexts/streaming.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ To run this you need to run it as a module via bytewax. diff --git a/examples/hamilton-tutorials/mpg-translation/MPGSimple.ipynb b/examples/hamilton-tutorials/mpg-translation/MPGSimple.ipynb index 16ae419dd..3c0f674e7 100644 --- a/examples/hamilton-tutorials/mpg-translation/MPGSimple.ipynb +++ b/examples/hamilton-tutorials/mpg-translation/MPGSimple.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/hamilton-tutorials/mpg-translation/MPGSimpleAdvancedTarget.ipynb b/examples/hamilton-tutorials/mpg-translation/MPGSimpleAdvancedTarget.ipynb index 048adfea4..8b4c47319 100644 --- a/examples/hamilton-tutorials/mpg-translation/MPGSimpleAdvancedTarget.ipynb +++ b/examples/hamilton-tutorials/mpg-translation/MPGSimpleAdvancedTarget.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/hamilton-tutorials/mpg-translation/MPGSimpleTarget.ipynb b/examples/hamilton-tutorials/mpg-translation/MPGSimpleTarget.ipynb index 45dbe2e5c..752953e1e 100644 --- a/examples/hamilton-tutorials/mpg-translation/MPGSimpleTarget.ipynb +++ b/examples/hamilton-tutorials/mpg-translation/MPGSimpleTarget.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/hamilton-tutorials/mpg-translation/ProceduralMPGNotebook.ipynb b/examples/hamilton-tutorials/mpg-translation/ProceduralMPGNotebook.ipynb index 9b8900e52..9402b6078 100644 --- a/examples/hamilton-tutorials/mpg-translation/ProceduralMPGNotebook.ipynb +++ b/examples/hamilton-tutorials/mpg-translation/ProceduralMPGNotebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/hamilton-tutorials/mpg-translation/README.md b/examples/hamilton-tutorials/mpg-translation/README.md index c12bad921..a01432cc7 100644 --- a/examples/hamilton-tutorials/mpg-translation/README.md +++ b/examples/hamilton-tutorials/mpg-translation/README.md @@ -1,3 +1,22 @@ + + # Translation exercise This is a tutorial that one can use to help people grok Apache Hamilton. diff --git a/examples/hamilton-tutorials/mpg-translation/mpg_pipeline.py b/examples/hamilton-tutorials/mpg-translation/mpg_pipeline.py index 0cbdba873..0f2f8c750 100644 --- a/examples/hamilton-tutorials/mpg-translation/mpg_pipeline.py +++ b/examples/hamilton-tutorials/mpg-translation/mpg_pipeline.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np # noqa: F401 import pandas as pd # noqa: F401 from sklearn.linear_model import LinearRegression # noqa: F401 diff --git a/examples/hamilton-tutorials/mpg-translation/mpg_pipeline_advanced_target.py b/examples/hamilton-tutorials/mpg-translation/mpg_pipeline_advanced_target.py index a17cd5215..5848b7ba7 100644 --- a/examples/hamilton-tutorials/mpg-translation/mpg_pipeline_advanced_target.py +++ b/examples/hamilton-tutorials/mpg-translation/mpg_pipeline_advanced_target.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import pandas as pd from sklearn.linear_model import LinearRegression diff --git a/examples/hamilton-tutorials/mpg-translation/mpg_pipeline_target.py b/examples/hamilton-tutorials/mpg-translation/mpg_pipeline_target.py index f021103db..593a5a0b8 100644 --- a/examples/hamilton-tutorials/mpg-translation/mpg_pipeline_target.py +++ b/examples/hamilton-tutorials/mpg-translation/mpg_pipeline_target.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import pandas as pd from sklearn.linear_model import LinearRegression diff --git a/examples/hamilton-tutorials/mpg-translation/mpg_script.py b/examples/hamilton-tutorials/mpg-translation/mpg_script.py index d213e4eb0..2cc5ffc89 100644 --- a/examples/hamilton-tutorials/mpg-translation/mpg_script.py +++ b/examples/hamilton-tutorials/mpg-translation/mpg_script.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import pandas as pd from sklearn.linear_model import LinearRegression diff --git a/examples/hamilton_ui/README.md b/examples/hamilton_ui/README.md index ec414c028..ae6e7fbf6 100644 --- a/examples/hamilton_ui/README.md +++ b/examples/hamilton_ui/README.md @@ -1,3 +1,22 @@ + + # Apache Hamilton UI - Machine learning example Learn how to use the `HamiltonTracker` and the Apache Hamilton UI to track a simple machine learning pipeline. diff --git a/examples/hamilton_ui/components/feature_transforms.py b/examples/hamilton_ui/components/feature_transforms.py index 829d5b8f1..59857288b 100644 --- a/examples/hamilton_ui/components/feature_transforms.py +++ b/examples/hamilton_ui/components/feature_transforms.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Module to transform iris data into features. """ diff --git a/examples/hamilton_ui/components/iris_loader.py b/examples/hamilton_ui/components/iris_loader.py index 5aa75408a..9db00b93a 100644 --- a/examples/hamilton_ui/components/iris_loader.py +++ b/examples/hamilton_ui/components/iris_loader.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Module to load iris data. """ diff --git a/examples/hamilton_ui/components/model_fitting.py b/examples/hamilton_ui/components/model_fitting.py index 778436312..f12b57bc7 100644 --- a/examples/hamilton_ui/components/model_fitting.py +++ b/examples/hamilton_ui/components/model_fitting.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """This module contains basic code for model fitting.""" from typing import Dict diff --git a/examples/hamilton_ui/components/models.py b/examples/hamilton_ui/components/models.py index 6c4e056de..403b39b7a 100644 --- a/examples/hamilton_ui/components/models.py +++ b/examples/hamilton_ui/components/models.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """This module contains specific incarnations of models.""" from sklearn import base diff --git a/examples/hamilton_ui/notebook.ipynb b/examples/hamilton_ui/notebook.ipynb index a14664cfd..1ab079965 100644 --- a/examples/hamilton_ui/notebook.ipynb +++ b/examples/hamilton_ui/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/hamilton_ui/run.py b/examples/hamilton_ui/run.py index 2799d78aa..e1fbb711a 100644 --- a/examples/hamilton_ui/run.py +++ b/examples/hamilton_ui/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import click from components import feature_transforms, iris_loader, models diff --git a/examples/hello_world/README.md b/examples/hello_world/README.md index d2fcf27c1..bb4027b0e 100644 --- a/examples/hello_world/README.md +++ b/examples/hello_world/README.md @@ -1,3 +1,22 @@ + + # Classic Apache Hamilton Hello World In this example we show you how to create a simple hello world dataflow that diff --git a/examples/hello_world/my_dag.png b/examples/hello_world/my_dag.png index 77e64e425..4f9bb2925 100644 Binary files a/examples/hello_world/my_dag.png and b/examples/hello_world/my_dag.png differ diff --git a/examples/hello_world/my_functions.py b/examples/hello_world/my_functions.py index b54f6b171..c57618842 100644 --- a/examples/hello_world/my_functions.py +++ b/examples/hello_world/my_functions.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd """ diff --git a/examples/hello_world/my_notebook.ipynb b/examples/hello_world/my_notebook.ipynb index 8e9edd954..b380572cf 100644 --- a/examples/hello_world/my_notebook.ipynb +++ b/examples/hello_world/my_notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/hello_world/my_script.py b/examples/hello_world/my_script.py index 39eb65145..8822fc73d 100644 --- a/examples/hello_world/my_script.py +++ b/examples/hello_world/my_script.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging import sys diff --git a/examples/ibis/feature_engineering/README.md b/examples/ibis/feature_engineering/README.md index 1ce49a1b9..94bef2acf 100644 --- a/examples/ibis/feature_engineering/README.md +++ b/examples/ibis/feature_engineering/README.md @@ -1,3 +1,22 @@ + + # Ibis + Apache Hamilton [Ibis](https://ibis-project.org/) is a portable dataframe library to write procedural data transformations in Python and be able to execute them directly on various SQL backends (DuckDB, Snowflake, Postgres, Flink, see [full list](https://ibis-project.org/support_matrix)). Apache Hamilton provides a declarative way to define testable, modular, self-documenting dataflows, that encode lineage and metadata. diff --git a/examples/ibis/feature_engineering/column_dataflow.py b/examples/ibis/feature_engineering/column_dataflow.py index 92ca41c2c..fac47c16f 100644 --- a/examples/ibis/feature_engineering/column_dataflow.py +++ b/examples/ibis/feature_engineering/column_dataflow.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Optional import ibis diff --git a/examples/ibis/feature_engineering/run.py b/examples/ibis/feature_engineering/run.py index 1682e5622..e07bbb411 100644 --- a/examples/ibis/feature_engineering/run.py +++ b/examples/ibis/feature_engineering/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from hamilton import driver from hamilton.plugins.h_tqdm import ProgressBar diff --git a/examples/ibis/feature_engineering/table_dataflow.py b/examples/ibis/feature_engineering/table_dataflow.py index 453f48c33..b70aecf08 100644 --- a/examples/ibis/feature_engineering/table_dataflow.py +++ b/examples/ibis/feature_engineering/table_dataflow.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Optional import ibis diff --git a/examples/ibis/jaffle_shop/README.md b/examples/ibis/jaffle_shop/README.md index 0ca57dbfb..9106156e7 100644 --- a/examples/ibis/jaffle_shop/README.md +++ b/examples/ibis/jaffle_shop/README.md @@ -1,3 +1,22 @@ + + # Jaffle shop This repository is a reimplementation of the canonical [dbt example jaffle_shop](https://github.com/dbt-labs/jaffle_shop). It illustrates data transformations for an ecommerce store. diff --git a/examples/ibis/jaffle_shop/dataflows/customer_flow.py b/examples/ibis/jaffle_shop/dataflows/customer_flow.py index 1600cf890..e4813558c 100644 --- a/examples/ibis/jaffle_shop/dataflows/customer_flow.py +++ b/examples/ibis/jaffle_shop/dataflows/customer_flow.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import ibis import ibis.expr.types as ir diff --git a/examples/ibis/jaffle_shop/dataflows/order_flow.py b/examples/ibis/jaffle_shop/dataflows/order_flow.py index 896bba43c..29b3219b6 100644 --- a/examples/ibis/jaffle_shop/dataflows/order_flow.py +++ b/examples/ibis/jaffle_shop/dataflows/order_flow.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import ibis import ibis.expr.types as ir diff --git a/examples/ibis/jaffle_shop/dataflows/staging.py b/examples/ibis/jaffle_shop/dataflows/staging.py index e1b663a1e..8ddcb9d9f 100644 --- a/examples/ibis/jaffle_shop/dataflows/staging.py +++ b/examples/ibis/jaffle_shop/dataflows/staging.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import ibis import ibis.expr.types as ir diff --git a/examples/ibis/jaffle_shop/run.py b/examples/ibis/jaffle_shop/run.py index 4346648d5..393caddf7 100644 --- a/examples/ibis/jaffle_shop/run.py +++ b/examples/ibis/jaffle_shop/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import ibis # import dataflow modules diff --git a/examples/ibisml/README.md b/examples/ibisml/README.md index 2af3a2f63..aa44a5ef2 100644 --- a/examples/ibisml/README.md +++ b/examples/ibisml/README.md @@ -1 +1,20 @@ + + # IbisML + Apache Hamilton [WIP] diff --git a/examples/ibisml/model_training.py b/examples/ibisml/model_training.py index 2beb019f3..9eb85d687 100644 --- a/examples/ibisml/model_training.py +++ b/examples/ibisml/model_training.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import ibis import ibis.expr.types as ir import ibisml diff --git a/examples/ibisml/run.py b/examples/ibisml/run.py index 6822c6775..e9ef3de71 100644 --- a/examples/ibisml/run.py +++ b/examples/ibisml/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from hamilton import driver from hamilton.execution.executors import SynchronousLocalTaskExecutor from hamilton.plugins.h_tqdm import ProgressBar diff --git a/examples/ibisml/table_dataflow.py b/examples/ibisml/table_dataflow.py index a75d8ade1..2e7ec7a9b 100644 --- a/examples/ibisml/table_dataflow.py +++ b/examples/ibisml/table_dataflow.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Optional import ibis diff --git a/examples/jupyter_notebook_magic/README.md b/examples/jupyter_notebook_magic/README.md index e6d866ac2..d584b89ba 100644 --- a/examples/jupyter_notebook_magic/README.md +++ b/examples/jupyter_notebook_magic/README.md @@ -1,3 +1,22 @@ + + # Apache Hamilton notebook extension One of the best part about notebooks is the ability to execute and immediately inspect results. They provide a "read-eval-print" loop (REPL) coding experience. However, the way Apache Hamilton separates dataflow definition (functions in a module) from execution (building and executing a driver) creates an extra step that can slowdown this loop. diff --git a/examples/jupyter_notebook_magic/example.ipynb b/examples/jupyter_notebook_magic/example.ipynb index 648ba23b8..98dc8643b 100644 --- a/examples/jupyter_notebook_magic/example.ipynb +++ b/examples/jupyter_notebook_magic/example.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": 1, @@ -96,7 +103,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[0;31mDocstring:\u001B[0m\n", + "\u001b[0;31mDocstring:\u001b[0m\n", "::\n", "\n", " %cell_to_module [-m [MODULE_NAME]] [-d [DISPLAY]] [--display_cache]\n", @@ -150,7 +157,7 @@ " -w <[WRITE_TO_FILE]>, --write_to_file <[WRITE_TO_FILE]>\n", " Write cell content to a file. The argument is the file\n", " path; else write to {module_name}.py\n", - "\u001B[0;31mFile:\u001B[0m ~/projects/hamilton/hamilton/plugins/jupyter_magic.py" + "\u001b[0;31mFile:\u001b[0m ~/projects/hamilton/hamilton/plugins/jupyter_magic.py" ] } ], diff --git a/examples/kedro/README.md b/examples/kedro/README.md index a651c51d7..4f35b4827 100644 --- a/examples/kedro/README.md +++ b/examples/kedro/README.md @@ -1,3 +1,22 @@ + + # Kedro & Apache Hamilton This repository compares how to build dataflows with Kedro and Apache Hamilton. diff --git a/examples/kedro/hamilton-code/README.md b/examples/kedro/hamilton-code/README.md index 8d37f114b..63f24a88b 100644 --- a/examples/kedro/hamilton-code/README.md +++ b/examples/kedro/hamilton-code/README.md @@ -1,3 +1,22 @@ + + # Apache Hamilton code ![](src/hamilton_code/all_functions.png) diff --git a/examples/kedro/hamilton-code/notebooks/interactive.ipynb b/examples/kedro/hamilton-code/notebooks/interactive.ipynb index b77ab60f0..7e754a9c3 100644 --- a/examples/kedro/hamilton-code/notebooks/interactive.ipynb +++ b/examples/kedro/hamilton-code/notebooks/interactive.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/kedro/hamilton-code/src/hamilton_code/data_processing.py b/examples/kedro/hamilton-code/src/hamilton_code/data_processing.py index 5aa507e76..0b11224ab 100644 --- a/examples/kedro/hamilton-code/src/hamilton_code/data_processing.py +++ b/examples/kedro/hamilton-code/src/hamilton_code/data_processing.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd diff --git a/examples/kedro/hamilton-code/src/hamilton_code/data_science.py b/examples/kedro/hamilton-code/src/hamilton_code/data_science.py index 1fec51450..0a6ef9ff1 100644 --- a/examples/kedro/hamilton-code/src/hamilton_code/data_science.py +++ b/examples/kedro/hamilton-code/src/hamilton_code/data_science.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging from typing import List, Union diff --git a/examples/kedro/hamilton-code/src/hamilton_code/run.py b/examples/kedro/hamilton-code/src/hamilton_code/run.py index f7b98d94a..6225c41e3 100644 --- a/examples/kedro/hamilton-code/src/hamilton_code/run.py +++ b/examples/kedro/hamilton-code/src/hamilton_code/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import data_processing import data_science diff --git a/examples/kedro/hamilton-code/tests/test_dataflow.py b/examples/kedro/hamilton-code/tests/test_dataflow.py index b23137e82..cf20b59f9 100644 --- a/examples/kedro/hamilton-code/tests/test_dataflow.py +++ b/examples/kedro/hamilton-code/tests/test_dataflow.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd import pytest diff --git a/examples/kedro/kedro-code/README.md b/examples/kedro/kedro-code/README.md index c6286ff6e..e7a22a2f8 100644 --- a/examples/kedro/kedro-code/README.md +++ b/examples/kedro/kedro-code/README.md @@ -1,3 +1,22 @@ + + # Kedro code ## Overview diff --git a/examples/kedro/kedro-code/conf/README.md b/examples/kedro/kedro-code/conf/README.md index b135e80c2..cc638d2ba 100644 --- a/examples/kedro/kedro-code/conf/README.md +++ b/examples/kedro/kedro-code/conf/README.md @@ -1,3 +1,22 @@ + + # What is this for? This folder should be used to store configuration files used by Kedro or by separate tools. diff --git a/examples/kedro/kedro-code/src/kedro_code/__main__.py b/examples/kedro/kedro-code/src/kedro_code/__main__.py index 7fbfed8f1..67fbdb8c9 100644 --- a/examples/kedro/kedro-code/src/kedro_code/__main__.py +++ b/examples/kedro/kedro-code/src/kedro_code/__main__.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """kedro_code file for ensuring the package is executable as `kedro-code` and `python -m kedro_code` """ diff --git a/examples/kedro/kedro-code/src/kedro_code/pipeline_registry.py b/examples/kedro/kedro-code/src/kedro_code/pipeline_registry.py index 54a1cc3ea..cb450d4b0 100644 --- a/examples/kedro/kedro-code/src/kedro_code/pipeline_registry.py +++ b/examples/kedro/kedro-code/src/kedro_code/pipeline_registry.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Project pipelines.""" from typing import Dict diff --git a/examples/kedro/kedro-code/src/kedro_code/pipelines/data_processing/nodes.py b/examples/kedro/kedro-code/src/kedro_code/pipelines/data_processing/nodes.py index 1adcf3613..5f821d8f3 100755 --- a/examples/kedro/kedro-code/src/kedro_code/pipelines/data_processing/nodes.py +++ b/examples/kedro/kedro-code/src/kedro_code/pipelines/data_processing/nodes.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd diff --git a/examples/kedro/kedro-code/src/kedro_code/pipelines/data_processing/pipeline.py b/examples/kedro/kedro-code/src/kedro_code/pipelines/data_processing/pipeline.py index 058ad9eae..06350f50e 100755 --- a/examples/kedro/kedro-code/src/kedro_code/pipelines/data_processing/pipeline.py +++ b/examples/kedro/kedro-code/src/kedro_code/pipelines/data_processing/pipeline.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from kedro.pipeline import Pipeline, node, pipeline from .nodes import create_model_input_table, preprocess_companies, preprocess_shuttles diff --git a/examples/kedro/kedro-code/src/kedro_code/pipelines/data_science/nodes.py b/examples/kedro/kedro-code/src/kedro_code/pipelines/data_science/nodes.py index ef921e3bd..d2eea6986 100755 --- a/examples/kedro/kedro-code/src/kedro_code/pipelines/data_science/nodes.py +++ b/examples/kedro/kedro-code/src/kedro_code/pipelines/data_science/nodes.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging from typing import Dict, Tuple diff --git a/examples/kedro/kedro-code/src/kedro_code/pipelines/data_science/pipeline.py b/examples/kedro/kedro-code/src/kedro_code/pipelines/data_science/pipeline.py index ce4c8aeb3..30dcd9242 100755 --- a/examples/kedro/kedro-code/src/kedro_code/pipelines/data_science/pipeline.py +++ b/examples/kedro/kedro-code/src/kedro_code/pipelines/data_science/pipeline.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from kedro.pipeline import Pipeline, node, pipeline from .nodes import evaluate_model, split_data, train_model diff --git a/examples/kedro/kedro-code/src/kedro_code/settings.py b/examples/kedro/kedro-code/src/kedro_code/settings.py index abea0e2c6..f1e2e6c5d 100644 --- a/examples/kedro/kedro-code/src/kedro_code/settings.py +++ b/examples/kedro/kedro-code/src/kedro_code/settings.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Project settings. There is no need to edit this file unless you want to change values from the Kedro defaults. For further information, including these default values, see https://docs.kedro.org/en/stable/kedro_project_setup/settings.html.""" diff --git a/examples/kedro/kedro-code/tests/pipelines/data_science/test_pipeline.py b/examples/kedro/kedro-code/tests/pipelines/data_science/test_pipeline.py index ed0c7e1ab..abba84e72 100644 --- a/examples/kedro/kedro-code/tests/pipelines/data_science/test_pipeline.py +++ b/examples/kedro/kedro-code/tests/pipelines/data_science/test_pipeline.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging from math import ceil diff --git a/examples/kedro/kedro-code/tests/test_run.py b/examples/kedro/kedro-code/tests/test_run.py index 158f7952f..540c5bb51 100644 --- a/examples/kedro/kedro-code/tests/test_run.py +++ b/examples/kedro/kedro-code/tests/test_run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This module contains an example test. diff --git a/examples/kedro/kedro-plugin/README.md b/examples/kedro/kedro-plugin/README.md index d6aa4b65e..b75135270 100644 --- a/examples/kedro/kedro-plugin/README.md +++ b/examples/kedro/kedro-plugin/README.md @@ -1,3 +1,22 @@ + + # Kedro plugin ## Content diff --git a/examples/kedro/kedro-plugin/kedro_to_hamilton.ipynb b/examples/kedro/kedro-plugin/kedro_to_hamilton.ipynb index bcb0e6e0a..cc0cc7b29 100644 --- a/examples/kedro/kedro-plugin/kedro_to_hamilton.ipynb +++ b/examples/kedro/kedro-plugin/kedro_to_hamilton.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, @@ -700,7 +707,7 @@ "\n" ], "text/plain": [ - "\u001B[2;36m[05/22/24 16:10:54]\u001B[0m\u001B[2;36m \u001B[0m\u001B[34mINFO \u001B[0m Loading data from \u001B[38;5;208mcompanies\u001B[0m \u001B[1m(\u001B[0mCSVDataset\u001B[1m)\u001B[0m\u001B[33m...\u001B[0m \u001B]8;id=554145;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001B\\\u001B[2mdata_catalog.py\u001B[0m\u001B]8;;\u001B\\\u001B[2m:\u001B[0m\u001B]8;id=747028;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001B\\\u001B[2m483\u001B[0m\u001B]8;;\u001B\\\n" + "\u001b[2;36m[05/22/24 16:10:54]\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Loading data from \u001b[38;5;208mcompanies\u001b[0m \u001b[1m(\u001b[0mCSVDataset\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=554145;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=747028;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001b\\\u001b[2m483\u001b[0m\u001b]8;;\u001b\\\n" ] }, "metadata": {}, @@ -713,7 +720,7 @@ "\n" ], "text/plain": [ - "\u001B[2;36m \u001B[0m\u001B[2;36m \u001B[0m\u001B[34mINFO \u001B[0m Loading data from \u001B[38;5;208mreviews\u001B[0m \u001B[1m(\u001B[0mCSVDataset\u001B[1m)\u001B[0m\u001B[33m...\u001B[0m \u001B]8;id=756925;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001B\\\u001B[2mdata_catalog.py\u001B[0m\u001B]8;;\u001B\\\u001B[2m:\u001B[0m\u001B]8;id=7965;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001B\\\u001B[2m483\u001B[0m\u001B]8;;\u001B\\\n" + "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Loading data from \u001b[38;5;208mreviews\u001b[0m \u001b[1m(\u001b[0mCSVDataset\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=756925;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=7965;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001b\\\u001b[2m483\u001b[0m\u001b]8;;\u001b\\\n" ] }, "metadata": {}, @@ -726,7 +733,7 @@ "\n" ], "text/plain": [ - "\u001B[2;36m \u001B[0m\u001B[2;36m \u001B[0m\u001B[34mINFO \u001B[0m Loading data from \u001B[38;5;208mshuttles\u001B[0m \u001B[1m(\u001B[0mExcelDataset\u001B[1m)\u001B[0m\u001B[33m...\u001B[0m \u001B]8;id=791772;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001B\\\u001B[2mdata_catalog.py\u001B[0m\u001B]8;;\u001B\\\u001B[2m:\u001B[0m\u001B]8;id=315027;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001B\\\u001B[2m483\u001B[0m\u001B]8;;\u001B\\\n" + "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Loading data from \u001b[38;5;208mshuttles\u001b[0m \u001b[1m(\u001b[0mExcelDataset\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=791772;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=315027;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001b\\\u001b[2m483\u001b[0m\u001b]8;;\u001b\\\n" ] }, "metadata": {}, @@ -915,41 +922,41 @@ "text/plain": [ "\n", " shuttle_location shuttle_type engine_type engine_vendor \\\n", - "\u001B[1;36m0\u001B[0m Sao Tome and Principe Type V5 Plasma ThetaBase Services \n", - "\u001B[1;36m1\u001B[0m Wallis and Futuna Type V2 Plasma ThetaBase Services \n", - "\u001B[1;36m2\u001B[0m Niue Type F5 Quantum ThetaBase Services \n", - "\u001B[1;36m3\u001B[0m Malta Type V2 Quantum ThetaBase Services \n", - "\u001B[1;36m4\u001B[0m Malta Type V2 Plasma ThetaBase Services \n", + "\u001b[1;36m0\u001b[0m Sao Tome and Principe Type V5 Plasma ThetaBase Services \n", + "\u001b[1;36m1\u001b[0m Wallis and Futuna Type V2 Plasma ThetaBase Services \n", + "\u001b[1;36m2\u001b[0m Niue Type F5 Quantum ThetaBase Services \n", + "\u001b[1;36m3\u001b[0m Malta Type V2 Quantum ThetaBase Services \n", + "\u001b[1;36m4\u001b[0m Malta Type V2 Plasma ThetaBase Services \n", "\n", " engines passenger_capacity cancellation_policy crew d_check_complete \\\n", - "\u001B[1;36m0\u001B[0m \u001B[1;36m2.0\u001B[0m \u001B[1;36m4\u001B[0m moderate \u001B[1;36m2.0\u001B[0m \u001B[3;91mFalse\u001B[0m \n", - "\u001B[1;36m1\u001B[0m \u001B[1;36m3.0\u001B[0m \u001B[1;36m5\u001B[0m moderate \u001B[1;36m3.0\u001B[0m \u001B[3;91mFalse\u001B[0m \n", - "\u001B[1;36m2\u001B[0m \u001B[1;36m1.0\u001B[0m \u001B[1;36m2\u001B[0m strict \u001B[1;36m1.0\u001B[0m \u001B[3;92mTrue\u001B[0m \n", - "\u001B[1;36m3\u001B[0m \u001B[1;36m1.0\u001B[0m \u001B[1;36m2\u001B[0m moderate \u001B[1;36m1.0\u001B[0m \u001B[3;91mFalse\u001B[0m \n", - "\u001B[1;36m4\u001B[0m \u001B[1;36m5.0\u001B[0m \u001B[1;36m10\u001B[0m strict \u001B[1;36m5.0\u001B[0m \u001B[3;91mFalse\u001B[0m \n", + "\u001b[1;36m0\u001b[0m \u001b[1;36m2.0\u001b[0m \u001b[1;36m4\u001b[0m moderate \u001b[1;36m2.0\u001b[0m \u001b[3;91mFalse\u001b[0m \n", + "\u001b[1;36m1\u001b[0m \u001b[1;36m3.0\u001b[0m \u001b[1;36m5\u001b[0m moderate \u001b[1;36m3.0\u001b[0m \u001b[3;91mFalse\u001b[0m \n", + "\u001b[1;36m2\u001b[0m \u001b[1;36m1.0\u001b[0m \u001b[1;36m2\u001b[0m strict \u001b[1;36m1.0\u001b[0m \u001b[3;92mTrue\u001b[0m \n", + "\u001b[1;36m3\u001b[0m \u001b[1;36m1.0\u001b[0m \u001b[1;36m2\u001b[0m moderate \u001b[1;36m1.0\u001b[0m \u001b[3;91mFalse\u001b[0m \n", + "\u001b[1;36m4\u001b[0m \u001b[1;36m5.0\u001b[0m \u001b[1;36m10\u001b[0m strict \u001b[1;36m5.0\u001b[0m \u001b[3;91mFalse\u001b[0m \n", "\n", - " moon_clearance_complete \u001B[33m...\u001B[0m review_scores_crew review_scores_location \\\n", - "\u001B[1;36m0\u001B[0m \u001B[3;91mFalse\u001B[0m \u001B[33m...\u001B[0m \u001B[1;36m9.0\u001B[0m \u001B[1;36m9.0\u001B[0m \n", - "\u001B[1;36m1\u001B[0m \u001B[3;91mFalse\u001B[0m \u001B[33m...\u001B[0m \u001B[1;36m10.0\u001B[0m \u001B[1;36m10.0\u001B[0m \n", - "\u001B[1;36m2\u001B[0m \u001B[3;91mFalse\u001B[0m \u001B[33m...\u001B[0m \u001B[1;36m10.0\u001B[0m \u001B[1;36m10.0\u001B[0m \n", - "\u001B[1;36m3\u001B[0m \u001B[3;91mFalse\u001B[0m \u001B[33m...\u001B[0m \u001B[1;36m10.0\u001B[0m \u001B[1;36m9.0\u001B[0m \n", - "\u001B[1;36m4\u001B[0m \u001B[3;91mFalse\u001B[0m \u001B[33m...\u001B[0m \u001B[1;36m10.0\u001B[0m \u001B[1;36m9.0\u001B[0m \n", + " moon_clearance_complete \u001b[33m...\u001b[0m review_scores_crew review_scores_location \\\n", + "\u001b[1;36m0\u001b[0m \u001b[3;91mFalse\u001b[0m \u001b[33m...\u001b[0m \u001b[1;36m9.0\u001b[0m \u001b[1;36m9.0\u001b[0m \n", + "\u001b[1;36m1\u001b[0m \u001b[3;91mFalse\u001b[0m \u001b[33m...\u001b[0m \u001b[1;36m10.0\u001b[0m \u001b[1;36m10.0\u001b[0m \n", + "\u001b[1;36m2\u001b[0m \u001b[3;91mFalse\u001b[0m \u001b[33m...\u001b[0m \u001b[1;36m10.0\u001b[0m \u001b[1;36m10.0\u001b[0m \n", + "\u001b[1;36m3\u001b[0m \u001b[3;91mFalse\u001b[0m \u001b[33m...\u001b[0m \u001b[1;36m10.0\u001b[0m \u001b[1;36m9.0\u001b[0m \n", + "\u001b[1;36m4\u001b[0m \u001b[3;91mFalse\u001b[0m \u001b[33m...\u001b[0m \u001b[1;36m10.0\u001b[0m \u001b[1;36m9.0\u001b[0m \n", "\n", " review_scores_price number_of_reviews reviews_per_month id \\\n", - "\u001B[1;36m0\u001B[0m \u001B[1;36m9.0\u001B[0m \u001B[1;36m26\u001B[0m \u001B[1;36m0.77\u001B[0m \u001B[1;36m32413\u001B[0m \n", - "\u001B[1;36m1\u001B[0m \u001B[1;36m9.0\u001B[0m \u001B[1;36m61\u001B[0m \u001B[1;36m0.62\u001B[0m \u001B[1;36m14122\u001B[0m \n", - "\u001B[1;36m2\u001B[0m \u001B[1;36m10.0\u001B[0m \u001B[1;36m467\u001B[0m \u001B[1;36m4.66\u001B[0m \u001B[1;36m47761\u001B[0m \n", - "\u001B[1;36m3\u001B[0m \u001B[1;36m9.0\u001B[0m \u001B[1;36m318\u001B[0m \u001B[1;36m3.22\u001B[0m \u001B[1;36m26648\u001B[0m \n", - "\u001B[1;36m4\u001B[0m \u001B[1;36m10.0\u001B[0m \u001B[1;36m22\u001B[0m \u001B[1;36m0.29\u001B[0m \u001B[1;36m26648\u001B[0m \n", + "\u001b[1;36m0\u001b[0m \u001b[1;36m9.0\u001b[0m \u001b[1;36m26\u001b[0m \u001b[1;36m0.77\u001b[0m \u001b[1;36m32413\u001b[0m \n", + "\u001b[1;36m1\u001b[0m \u001b[1;36m9.0\u001b[0m \u001b[1;36m61\u001b[0m \u001b[1;36m0.62\u001b[0m \u001b[1;36m14122\u001b[0m \n", + "\u001b[1;36m2\u001b[0m \u001b[1;36m10.0\u001b[0m \u001b[1;36m467\u001b[0m \u001b[1;36m4.66\u001b[0m \u001b[1;36m47761\u001b[0m \n", + "\u001b[1;36m3\u001b[0m \u001b[1;36m9.0\u001b[0m \u001b[1;36m318\u001b[0m \u001b[1;36m3.22\u001b[0m \u001b[1;36m26648\u001b[0m \n", + "\u001b[1;36m4\u001b[0m \u001b[1;36m10.0\u001b[0m \u001b[1;36m22\u001b[0m \u001b[1;36m0.29\u001b[0m \u001b[1;36m26648\u001b[0m \n", "\n", " company_rating company_location total_fleet_count iata_approved \n", - "\u001B[1;36m0\u001B[0m \u001B[1;36m1.0\u001B[0m Faroe Islands \u001B[1;36m1.0\u001B[0m \u001B[3;91mFalse\u001B[0m \n", - "\u001B[1;36m1\u001B[0m \u001B[1;36m1.0\u001B[0m Malta \u001B[1;36m1.0\u001B[0m \u001B[3;92mTrue\u001B[0m \n", - "\u001B[1;36m2\u001B[0m \u001B[1;36m1.0\u001B[0m Niue \u001B[1;36m2.0\u001B[0m \u001B[3;91mFalse\u001B[0m \n", - "\u001B[1;36m3\u001B[0m \u001B[1;36m1.0\u001B[0m Niue \u001B[1;36m2.0\u001B[0m \u001B[3;92mTrue\u001B[0m \n", - "\u001B[1;36m4\u001B[0m \u001B[1;36m1.0\u001B[0m Niue \u001B[1;36m2.0\u001B[0m \u001B[3;92mTrue\u001B[0m \n", + "\u001b[1;36m0\u001b[0m \u001b[1;36m1.0\u001b[0m Faroe Islands \u001b[1;36m1.0\u001b[0m \u001b[3;91mFalse\u001b[0m \n", + "\u001b[1;36m1\u001b[0m \u001b[1;36m1.0\u001b[0m Malta \u001b[1;36m1.0\u001b[0m \u001b[3;92mTrue\u001b[0m \n", + "\u001b[1;36m2\u001b[0m \u001b[1;36m1.0\u001b[0m Niue \u001b[1;36m2.0\u001b[0m \u001b[3;91mFalse\u001b[0m \n", + "\u001b[1;36m3\u001b[0m \u001b[1;36m1.0\u001b[0m Niue \u001b[1;36m2.0\u001b[0m \u001b[3;92mTrue\u001b[0m \n", + "\u001b[1;36m4\u001b[0m \u001b[1;36m1.0\u001b[0m Niue \u001b[1;36m2.0\u001b[0m \u001b[3;92mTrue\u001b[0m \n", "\n", - "\u001B[1m[\u001B[0m\u001B[1;36m5\u001B[0m rows x \u001B[1;36m27\u001B[0m columns\u001B[1m]\u001B[0m" + "\u001b[1m[\u001b[0m\u001b[1;36m5\u001b[0m rows x \u001b[1;36m27\u001b[0m columns\u001b[1m]\u001b[0m" ] }, "execution_count": 5, @@ -1009,80 +1016,80 @@ "data": { "text/plain": [ "\n", - "\u001B[1m{\u001B[0m\n", - " \u001B[32m'model_input_table__parquet'\u001B[0m: \u001B[1m{\u001B[0m\n", - " \u001B[32m'file_metadata'\u001B[0m: \u001B[1m{\u001B[0m\n", - " \u001B[32m'size'\u001B[0m: \u001B[1;36m215150\u001B[0m,\n", - " \u001B[32m'path'\u001B[0m: \u001B[32m'../kedro-code/data/03_primary/model_input_table.pq'\u001B[0m,\n", - " \u001B[32m'last_modified'\u001B[0m: \u001B[1;36m1716408659.312288\u001B[0m,\n", - " \u001B[32m'timestamp'\u001B[0m: \u001B[1;36m1716423059.328785\u001B[0m,\n", - " \u001B[32m'scheme'\u001B[0m: \u001B[32m''\u001B[0m,\n", - " \u001B[32m'notes'\u001B[0m: \u001B[32m''\u001B[0m\n", - " \u001B[1m}\u001B[0m,\n", - " \u001B[32m'dataframe_metadata'\u001B[0m: \u001B[1m{\u001B[0m\n", - " \u001B[32m'rows'\u001B[0m: \u001B[1;36m6027\u001B[0m,\n", - " \u001B[32m'columns'\u001B[0m: \u001B[1;36m27\u001B[0m,\n", - " \u001B[32m'column_names'\u001B[0m: \u001B[1m[\u001B[0m\n", - " \u001B[32m'shuttle_location'\u001B[0m,\n", - " \u001B[32m'shuttle_type'\u001B[0m,\n", - " \u001B[32m'engine_type'\u001B[0m,\n", - " \u001B[32m'engine_vendor'\u001B[0m,\n", - " \u001B[32m'engines'\u001B[0m,\n", - " \u001B[32m'passenger_capacity'\u001B[0m,\n", - " \u001B[32m'cancellation_policy'\u001B[0m,\n", - " \u001B[32m'crew'\u001B[0m,\n", - " \u001B[32m'd_check_complete'\u001B[0m,\n", - " \u001B[32m'moon_clearance_complete'\u001B[0m,\n", - " \u001B[32m'price'\u001B[0m,\n", - " \u001B[32m'company_id'\u001B[0m,\n", - " \u001B[32m'shuttle_id'\u001B[0m,\n", - " \u001B[32m'review_scores_rating'\u001B[0m,\n", - " \u001B[32m'review_scores_comfort'\u001B[0m,\n", - " \u001B[32m'review_scores_amenities'\u001B[0m,\n", - " \u001B[32m'review_scores_trip'\u001B[0m,\n", - " \u001B[32m'review_scores_crew'\u001B[0m,\n", - " \u001B[32m'review_scores_location'\u001B[0m,\n", - " \u001B[32m'review_scores_price'\u001B[0m,\n", - " \u001B[32m'number_of_reviews'\u001B[0m,\n", - " \u001B[32m'reviews_per_month'\u001B[0m,\n", - " \u001B[32m'id'\u001B[0m,\n", - " \u001B[32m'company_rating'\u001B[0m,\n", - " \u001B[32m'company_location'\u001B[0m,\n", - " \u001B[32m'total_fleet_count'\u001B[0m,\n", - " \u001B[32m'iata_approved'\u001B[0m\n", - " \u001B[1m]\u001B[0m,\n", - " \u001B[32m'datatypes'\u001B[0m: \u001B[1m[\u001B[0m\n", - " \u001B[32m'string'\u001B[0m,\n", - " \u001B[32m'string'\u001B[0m,\n", - " \u001B[32m'string'\u001B[0m,\n", - " \u001B[32m'string'\u001B[0m,\n", - " \u001B[32m'Int64'\u001B[0m,\n", - " \u001B[32m'Int64'\u001B[0m,\n", - " \u001B[32m'string'\u001B[0m,\n", - " \u001B[32m'Int64'\u001B[0m,\n", - " \u001B[32m'boolean'\u001B[0m,\n", - " \u001B[32m'boolean'\u001B[0m,\n", - " \u001B[32m'float64'\u001B[0m,\n", - " \u001B[32m'Int64'\u001B[0m,\n", - " \u001B[32m'Int64'\u001B[0m,\n", - " \u001B[32m'Float64'\u001B[0m,\n", - " \u001B[32m'Float64'\u001B[0m,\n", - " \u001B[32m'Float64'\u001B[0m,\n", - " \u001B[32m'Float64'\u001B[0m,\n", - " \u001B[32m'Float64'\u001B[0m,\n", - " \u001B[32m'Float64'\u001B[0m,\n", - " \u001B[32m'Float64'\u001B[0m,\n", - " \u001B[32m'Int64'\u001B[0m,\n", - " \u001B[32m'Float64'\u001B[0m,\n", - " \u001B[32m'Int64'\u001B[0m,\n", - " \u001B[32m'float64'\u001B[0m,\n", - " \u001B[32m'string'\u001B[0m,\n", - " \u001B[32m'Float64'\u001B[0m,\n", - " \u001B[32m'boolean'\u001B[0m\n", - " \u001B[1m]\u001B[0m\n", - " \u001B[1m}\u001B[0m\n", - " \u001B[1m}\u001B[0m\n", - "\u001B[1m}\u001B[0m" + "\u001b[1m{\u001b[0m\n", + " \u001b[32m'model_input_table__parquet'\u001b[0m: \u001b[1m{\u001b[0m\n", + " \u001b[32m'file_metadata'\u001b[0m: \u001b[1m{\u001b[0m\n", + " \u001b[32m'size'\u001b[0m: \u001b[1;36m215150\u001b[0m,\n", + " \u001b[32m'path'\u001b[0m: \u001b[32m'../kedro-code/data/03_primary/model_input_table.pq'\u001b[0m,\n", + " \u001b[32m'last_modified'\u001b[0m: \u001b[1;36m1716408659.312288\u001b[0m,\n", + " \u001b[32m'timestamp'\u001b[0m: \u001b[1;36m1716423059.328785\u001b[0m,\n", + " \u001b[32m'scheme'\u001b[0m: \u001b[32m''\u001b[0m,\n", + " \u001b[32m'notes'\u001b[0m: \u001b[32m''\u001b[0m\n", + " \u001b[1m}\u001b[0m,\n", + " \u001b[32m'dataframe_metadata'\u001b[0m: \u001b[1m{\u001b[0m\n", + " \u001b[32m'rows'\u001b[0m: \u001b[1;36m6027\u001b[0m,\n", + " \u001b[32m'columns'\u001b[0m: \u001b[1;36m27\u001b[0m,\n", + " \u001b[32m'column_names'\u001b[0m: \u001b[1m[\u001b[0m\n", + " \u001b[32m'shuttle_location'\u001b[0m,\n", + " \u001b[32m'shuttle_type'\u001b[0m,\n", + " \u001b[32m'engine_type'\u001b[0m,\n", + " \u001b[32m'engine_vendor'\u001b[0m,\n", + " \u001b[32m'engines'\u001b[0m,\n", + " \u001b[32m'passenger_capacity'\u001b[0m,\n", + " \u001b[32m'cancellation_policy'\u001b[0m,\n", + " \u001b[32m'crew'\u001b[0m,\n", + " \u001b[32m'd_check_complete'\u001b[0m,\n", + " \u001b[32m'moon_clearance_complete'\u001b[0m,\n", + " \u001b[32m'price'\u001b[0m,\n", + " \u001b[32m'company_id'\u001b[0m,\n", + " \u001b[32m'shuttle_id'\u001b[0m,\n", + " \u001b[32m'review_scores_rating'\u001b[0m,\n", + " \u001b[32m'review_scores_comfort'\u001b[0m,\n", + " \u001b[32m'review_scores_amenities'\u001b[0m,\n", + " \u001b[32m'review_scores_trip'\u001b[0m,\n", + " \u001b[32m'review_scores_crew'\u001b[0m,\n", + " \u001b[32m'review_scores_location'\u001b[0m,\n", + " \u001b[32m'review_scores_price'\u001b[0m,\n", + " \u001b[32m'number_of_reviews'\u001b[0m,\n", + " \u001b[32m'reviews_per_month'\u001b[0m,\n", + " \u001b[32m'id'\u001b[0m,\n", + " \u001b[32m'company_rating'\u001b[0m,\n", + " \u001b[32m'company_location'\u001b[0m,\n", + " \u001b[32m'total_fleet_count'\u001b[0m,\n", + " \u001b[32m'iata_approved'\u001b[0m\n", + " \u001b[1m]\u001b[0m,\n", + " \u001b[32m'datatypes'\u001b[0m: \u001b[1m[\u001b[0m\n", + " \u001b[32m'string'\u001b[0m,\n", + " \u001b[32m'string'\u001b[0m,\n", + " \u001b[32m'string'\u001b[0m,\n", + " \u001b[32m'string'\u001b[0m,\n", + " \u001b[32m'Int64'\u001b[0m,\n", + " \u001b[32m'Int64'\u001b[0m,\n", + " \u001b[32m'string'\u001b[0m,\n", + " \u001b[32m'Int64'\u001b[0m,\n", + " \u001b[32m'boolean'\u001b[0m,\n", + " \u001b[32m'boolean'\u001b[0m,\n", + " \u001b[32m'float64'\u001b[0m,\n", + " \u001b[32m'Int64'\u001b[0m,\n", + " \u001b[32m'Int64'\u001b[0m,\n", + " \u001b[32m'Float64'\u001b[0m,\n", + " \u001b[32m'Float64'\u001b[0m,\n", + " \u001b[32m'Float64'\u001b[0m,\n", + " \u001b[32m'Float64'\u001b[0m,\n", + " \u001b[32m'Float64'\u001b[0m,\n", + " \u001b[32m'Float64'\u001b[0m,\n", + " \u001b[32m'Float64'\u001b[0m,\n", + " \u001b[32m'Int64'\u001b[0m,\n", + " \u001b[32m'Float64'\u001b[0m,\n", + " \u001b[32m'Int64'\u001b[0m,\n", + " \u001b[32m'float64'\u001b[0m,\n", + " \u001b[32m'string'\u001b[0m,\n", + " \u001b[32m'Float64'\u001b[0m,\n", + " \u001b[32m'boolean'\u001b[0m\n", + " \u001b[1m]\u001b[0m\n", + " \u001b[1m}\u001b[0m\n", + " \u001b[1m}\u001b[0m\n", + "\u001b[1m}\u001b[0m" ] }, "execution_count": 6, @@ -1143,7 +1150,7 @@ "\n" ], "text/plain": [ - "\u001B[2;36m[05/22/24 16:10:59]\u001B[0m\u001B[2;36m \u001B[0m\u001B[34mINFO \u001B[0m Loading data from \u001B[38;5;208mshuttles\u001B[0m \u001B[1m(\u001B[0mExcelDataset\u001B[1m)\u001B[0m\u001B[33m...\u001B[0m \u001B]8;id=251889;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001B\\\u001B[2mdata_catalog.py\u001B[0m\u001B]8;;\u001B\\\u001B[2m:\u001B[0m\u001B]8;id=569579;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001B\\\u001B[2m483\u001B[0m\u001B]8;;\u001B\\\n" + "\u001b[2;36m[05/22/24 16:10:59]\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Loading data from \u001b[38;5;208mshuttles\u001b[0m \u001b[1m(\u001b[0mExcelDataset\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=251889;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=569579;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001b\\\u001b[2m483\u001b[0m\u001b]8;;\u001b\\\n" ] }, "metadata": {}, @@ -1156,7 +1163,7 @@ "\n" ], "text/plain": [ - "\u001B[2;36m[05/22/24 16:11:01]\u001B[0m\u001B[2;36m \u001B[0m\u001B[34mINFO \u001B[0m Loading data from \u001B[38;5;208mcompanies\u001B[0m \u001B[1m(\u001B[0mCSVDataset\u001B[1m)\u001B[0m\u001B[33m...\u001B[0m \u001B]8;id=698982;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001B\\\u001B[2mdata_catalog.py\u001B[0m\u001B]8;;\u001B\\\u001B[2m:\u001B[0m\u001B]8;id=247154;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001B\\\u001B[2m483\u001B[0m\u001B]8;;\u001B\\\n" + "\u001b[2;36m[05/22/24 16:11:01]\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Loading data from \u001b[38;5;208mcompanies\u001b[0m \u001b[1m(\u001b[0mCSVDataset\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=698982;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=247154;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001b\\\u001b[2m483\u001b[0m\u001b]8;;\u001b\\\n" ] }, "metadata": {}, @@ -1169,7 +1176,7 @@ "\n" ], "text/plain": [ - "\u001B[2;36m \u001B[0m\u001B[2;36m \u001B[0m\u001B[34mINFO \u001B[0m Loading data from \u001B[38;5;208mreviews\u001B[0m \u001B[1m(\u001B[0mCSVDataset\u001B[1m)\u001B[0m\u001B[33m...\u001B[0m \u001B]8;id=651853;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001B\\\u001B[2mdata_catalog.py\u001B[0m\u001B]8;;\u001B\\\u001B[2m:\u001B[0m\u001B]8;id=754005;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001B\\\u001B[2m483\u001B[0m\u001B]8;;\u001B\\\n" + "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Loading data from \u001b[38;5;208mreviews\u001b[0m \u001b[1m(\u001b[0mCSVDataset\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=651853;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=754005;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001b\\\u001b[2m483\u001b[0m\u001b]8;;\u001b\\\n" ] }, "metadata": {}, @@ -1182,7 +1189,7 @@ "\n" ], "text/plain": [ - "\u001B[2;36m[05/22/24 16:11:02]\u001B[0m\u001B[2;36m \u001B[0m\u001B[34mINFO \u001B[0m Saving data to \u001B[38;5;208mmodel_input_table\u001B[0m \u001B[1m(\u001B[0mParquetDataset\u001B[1m)\u001B[0m\u001B[33m...\u001B[0m \u001B]8;id=845498;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001B\\\u001B[2mdata_catalog.py\u001B[0m\u001B]8;;\u001B\\\u001B[2m:\u001B[0m\u001B]8;id=59594;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#525\u001B\\\u001B[2m525\u001B[0m\u001B]8;;\u001B\\\n" + "\u001b[2;36m[05/22/24 16:11:02]\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Saving data to \u001b[38;5;208mmodel_input_table\u001b[0m \u001b[1m(\u001b[0mParquetDataset\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=845498;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=59594;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#525\u001b\\\u001b[2m525\u001b[0m\u001b]8;;\u001b\\\n" ] }, "metadata": {}, @@ -1201,7 +1208,7 @@ { "data": { "text/plain": [ - "\u001B[1m{\u001B[0m\u001B[32m'model_input_table__parquet'\u001B[0m: \u001B[1m{\u001B[0m\u001B[32m'success'\u001B[0m: \u001B[3;92mTrue\u001B[0m\u001B[1m}\u001B[0m\u001B[1m}\u001B[0m" + "\u001b[1m{\u001b[0m\u001b[32m'model_input_table__parquet'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'success'\u001b[0m: \u001b[3;92mTrue\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m" ] }, "execution_count": 7, @@ -1264,7 +1271,7 @@ "\n" ], "text/plain": [ - "\u001B[2;36m \u001B[0m\u001B[2;36m \u001B[0m\u001B[34mINFO \u001B[0m Loading data from \u001B[38;5;208mshuttles\u001B[0m \u001B[1m(\u001B[0mExcelDataset\u001B[1m)\u001B[0m\u001B[33m...\u001B[0m \u001B]8;id=765638;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001B\\\u001B[2mdata_catalog.py\u001B[0m\u001B]8;;\u001B\\\u001B[2m:\u001B[0m\u001B]8;id=540969;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001B\\\u001B[2m483\u001B[0m\u001B]8;;\u001B\\\n" + "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Loading data from \u001b[38;5;208mshuttles\u001b[0m \u001b[1m(\u001b[0mExcelDataset\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=765638;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=540969;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001b\\\u001b[2m483\u001b[0m\u001b]8;;\u001b\\\n" ] }, "metadata": {}, @@ -1277,7 +1284,7 @@ "\n" ], "text/plain": [ - "\u001B[2;36m[05/22/24 16:11:04]\u001B[0m\u001B[2;36m \u001B[0m\u001B[34mINFO \u001B[0m Loading data from \u001B[38;5;208mcompanies\u001B[0m \u001B[1m(\u001B[0mCSVDataset\u001B[1m)\u001B[0m\u001B[33m...\u001B[0m \u001B]8;id=845539;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001B\\\u001B[2mdata_catalog.py\u001B[0m\u001B]8;;\u001B\\\u001B[2m:\u001B[0m\u001B]8;id=373818;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001B\\\u001B[2m483\u001B[0m\u001B]8;;\u001B\\\n" + "\u001b[2;36m[05/22/24 16:11:04]\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Loading data from \u001b[38;5;208mcompanies\u001b[0m \u001b[1m(\u001b[0mCSVDataset\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=845539;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=373818;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001b\\\u001b[2m483\u001b[0m\u001b]8;;\u001b\\\n" ] }, "metadata": {}, @@ -1290,7 +1297,7 @@ "\n" ], "text/plain": [ - "\u001B[2;36m \u001B[0m\u001B[2;36m \u001B[0m\u001B[34mINFO \u001B[0m Loading data from \u001B[38;5;208mreviews\u001B[0m \u001B[1m(\u001B[0mCSVDataset\u001B[1m)\u001B[0m\u001B[33m...\u001B[0m \u001B]8;id=673966;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001B\\\u001B[2mdata_catalog.py\u001B[0m\u001B]8;;\u001B\\\u001B[2m:\u001B[0m\u001B]8;id=380648;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001B\\\u001B[2m483\u001B[0m\u001B]8;;\u001B\\\n" + "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Loading data from \u001b[38;5;208mreviews\u001b[0m \u001b[1m(\u001b[0mCSVDataset\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=673966;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=380648;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001b\\\u001b[2m483\u001b[0m\u001b]8;;\u001b\\\n" ] }, "metadata": {}, @@ -1303,7 +1310,7 @@ "\n" ], "text/plain": [ - "\u001B[2;36m \u001B[0m\u001B[2;36m \u001B[0m\u001B[34mINFO \u001B[0m Saving data to \u001B[38;5;208mmodel_input_table\u001B[0m \u001B[1m(\u001B[0mParquetDataset\u001B[1m)\u001B[0m\u001B[33m...\u001B[0m \u001B]8;id=611834;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001B\\\u001B[2mdata_catalog.py\u001B[0m\u001B]8;;\u001B\\\u001B[2m:\u001B[0m\u001B]8;id=912002;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#525\u001B\\\u001B[2m525\u001B[0m\u001B]8;;\u001B\\\n" + "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Saving data to \u001b[38;5;208mmodel_input_table\u001b[0m \u001b[1m(\u001b[0mParquetDataset\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=611834;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=912002;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#525\u001b\\\u001b[2m525\u001b[0m\u001b]8;;\u001b\\\n" ] }, "metadata": {}, @@ -1322,7 +1329,7 @@ { "data": { "text/plain": [ - "\u001B[1m{\u001B[0m\u001B[32m'model_input_table__parquet'\u001B[0m: \u001B[1m{\u001B[0m\u001B[32m'success'\u001B[0m: \u001B[3;92mTrue\u001B[0m\u001B[1m}\u001B[0m\u001B[1m}\u001B[0m" + "\u001b[1m{\u001b[0m\u001b[32m'model_input_table__parquet'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'success'\u001b[0m: \u001b[3;92mTrue\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m" ] }, "execution_count": 8, @@ -1643,7 +1650,7 @@ "\n" ], "text/plain": [ - "\u001B[1m<\u001B[0m\u001B[1;95mhamilton.driver.Driver\u001B[0m\u001B[39m object at \u001B[0m\u001B[1;36m0x7f267841b450\u001B[0m\u001B[1m>\u001B[0m" + "\u001b[1m<\u001b[0m\u001b[1;95mhamilton.driver.Driver\u001b[0m\u001b[39m object at \u001b[0m\u001b[1;36m0x7f267841b450\u001b[0m\u001b[1m>\u001b[0m" ] }, "execution_count": 10, @@ -1695,10 +1702,10 @@ "\n" ], "text/plain": [ - "\u001B[2;36m[05/22/24 16:11:10]\u001B[0m\u001B[2;36m \u001B[0m\u001B[31mWARNING \u001B[0m \u001B]8;id=299905;file:///home/tjean/projects/dagworks/hamilton/ui/sdk/src/hamilton_sdk/adapters.py\u001B\\\u001B[2madapters.py\u001B[0m\u001B]8;;\u001B\\\u001B[2m:\u001B[0m\u001B]8;id=155714;file:///home/tjean/projects/dagworks/hamilton/ui/sdk/src/hamilton_sdk/adapters.py#163\u001B\\\u001B[2m163\u001B[0m\u001B]8;;\u001B\\\n", - "\u001B[2;36m \u001B[0m Capturing execution run. Results can be found at \u001B[2m \u001B[0m\n", - "\u001B[2;36m \u001B[0m \u001B[4;94mhttp://localhost:8242/dashboard/project/3/runs/32\u001B[0m \u001B[2m \u001B[0m\n", - "\u001B[2;36m \u001B[0m \u001B[2m \u001B[0m\n" + "\u001b[2;36m[05/22/24 16:11:10]\u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m \u001b]8;id=299905;file:///home/tjean/projects/dagworks/hamilton/ui/sdk/src/hamilton_sdk/adapters.py\u001b\\\u001b[2madapters.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=155714;file:///home/tjean/projects/dagworks/hamilton/ui/sdk/src/hamilton_sdk/adapters.py#163\u001b\\\u001b[2m163\u001b[0m\u001b]8;;\u001b\\\n", + "\u001b[2;36m \u001b[0m Capturing execution run. Results can be found at \u001b[2m \u001b[0m\n", + "\u001b[2;36m \u001b[0m \u001b[4;94mhttp://localhost:8242/dashboard/project/3/runs/32\u001b[0m \u001b[2m \u001b[0m\n", + "\u001b[2;36m \u001b[0m \u001b[2m \u001b[0m\n" ] }, "metadata": {}, @@ -1711,7 +1718,7 @@ "\n" ], "text/plain": [ - "\u001B[2;36m \u001B[0m\u001B[2;36m \u001B[0m\u001B[34mINFO \u001B[0m Loading data from \u001B[38;5;208mshuttles\u001B[0m \u001B[1m(\u001B[0mExcelDataset\u001B[1m)\u001B[0m\u001B[33m...\u001B[0m \u001B]8;id=976064;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001B\\\u001B[2mdata_catalog.py\u001B[0m\u001B]8;;\u001B\\\u001B[2m:\u001B[0m\u001B]8;id=997503;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001B\\\u001B[2m483\u001B[0m\u001B]8;;\u001B\\\n" + "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Loading data from \u001b[38;5;208mshuttles\u001b[0m \u001b[1m(\u001b[0mExcelDataset\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=976064;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=997503;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001b\\\u001b[2m483\u001b[0m\u001b]8;;\u001b\\\n" ] }, "metadata": {}, @@ -1724,7 +1731,7 @@ "\n" ], "text/plain": [ - "\u001B[2;36m[05/22/24 16:11:13]\u001B[0m\u001B[2;36m \u001B[0m\u001B[34mINFO \u001B[0m Loading data from \u001B[38;5;208mcompanies\u001B[0m \u001B[1m(\u001B[0mCSVDataset\u001B[1m)\u001B[0m\u001B[33m...\u001B[0m \u001B]8;id=134319;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001B\\\u001B[2mdata_catalog.py\u001B[0m\u001B]8;;\u001B\\\u001B[2m:\u001B[0m\u001B]8;id=446665;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001B\\\u001B[2m483\u001B[0m\u001B]8;;\u001B\\\n" + "\u001b[2;36m[05/22/24 16:11:13]\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Loading data from \u001b[38;5;208mcompanies\u001b[0m \u001b[1m(\u001b[0mCSVDataset\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=134319;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=446665;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001b\\\u001b[2m483\u001b[0m\u001b]8;;\u001b\\\n" ] }, "metadata": {}, @@ -1737,7 +1744,7 @@ "\n" ], "text/plain": [ - "\u001B[2;36m \u001B[0m\u001B[2;36m \u001B[0m\u001B[34mINFO \u001B[0m Loading data from \u001B[38;5;208mreviews\u001B[0m \u001B[1m(\u001B[0mCSVDataset\u001B[1m)\u001B[0m\u001B[33m...\u001B[0m \u001B]8;id=620527;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001B\\\u001B[2mdata_catalog.py\u001B[0m\u001B]8;;\u001B\\\u001B[2m:\u001B[0m\u001B]8;id=424729;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001B\\\u001B[2m483\u001B[0m\u001B]8;;\u001B\\\n" + "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Loading data from \u001b[38;5;208mreviews\u001b[0m \u001b[1m(\u001b[0mCSVDataset\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=620527;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=424729;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#483\u001b\\\u001b[2m483\u001b[0m\u001b]8;;\u001b\\\n" ] }, "metadata": {}, @@ -1750,7 +1757,7 @@ "\n" ], "text/plain": [ - "\u001B[2;36m \u001B[0m\u001B[2;36m \u001B[0m\u001B[34mINFO \u001B[0m Saving data to \u001B[38;5;208mmodel_input_table\u001B[0m \u001B[1m(\u001B[0mParquetDataset\u001B[1m)\u001B[0m\u001B[33m...\u001B[0m \u001B]8;id=102118;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001B\\\u001B[2mdata_catalog.py\u001B[0m\u001B]8;;\u001B\\\u001B[2m:\u001B[0m\u001B]8;id=182015;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#525\u001B\\\u001B[2m525\u001B[0m\u001B]8;;\u001B\\\n" + "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m Saving data to \u001b[38;5;208mmodel_input_table\u001b[0m \u001b[1m(\u001b[0mParquetDataset\u001b[1m)\u001b[0m\u001b[33m...\u001b[0m \u001b]8;id=102118;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py\u001b\\\u001b[2mdata_catalog.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=182015;file:///home/tjean/projects/dagworks/hamilton/examples/kedro/venv/lib/python3.11/site-packages/kedro/io/data_catalog.py#525\u001b\\\u001b[2m525\u001b[0m\u001b]8;;\u001b\\\n" ] }, "metadata": {}, @@ -1766,10 +1773,10 @@ "\n" ], "text/plain": [ - "\u001B[2;36m \u001B[0m\u001B[2;36m \u001B[0m\u001B[31mWARNING \u001B[0m \u001B]8;id=48567;file:///home/tjean/projects/dagworks/hamilton/ui/sdk/src/hamilton_sdk/adapters.py\u001B\\\u001B[2madapters.py\u001B[0m\u001B]8;;\u001B\\\u001B[2m:\u001B[0m\u001B]8;id=639017;file:///home/tjean/projects/dagworks/hamilton/ui/sdk/src/hamilton_sdk/adapters.py#352\u001B\\\u001B[2m352\u001B[0m\u001B]8;;\u001B\\\n", - "\u001B[2;36m \u001B[0m Captured execution run. Results can be found at \u001B[2m \u001B[0m\n", - "\u001B[2;36m \u001B[0m \u001B[4;94mhttp://localhost:8242/dashboard/project/3/runs/32\u001B[0m \u001B[2m \u001B[0m\n", - "\u001B[2;36m \u001B[0m \u001B[2m \u001B[0m\n" + "\u001b[2;36m \u001b[0m\u001b[2;36m \u001b[0m\u001b[31mWARNING \u001b[0m \u001b]8;id=48567;file:///home/tjean/projects/dagworks/hamilton/ui/sdk/src/hamilton_sdk/adapters.py\u001b\\\u001b[2madapters.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=639017;file:///home/tjean/projects/dagworks/hamilton/ui/sdk/src/hamilton_sdk/adapters.py#352\u001b\\\u001b[2m352\u001b[0m\u001b]8;;\u001b\\\n", + "\u001b[2;36m \u001b[0m Captured execution run. Results can be found at \u001b[2m \u001b[0m\n", + "\u001b[2;36m \u001b[0m \u001b[4;94mhttp://localhost:8242/dashboard/project/3/runs/32\u001b[0m \u001b[2m \u001b[0m\n", + "\u001b[2;36m \u001b[0m \u001b[2m \u001b[0m\n" ] }, "metadata": {}, diff --git a/examples/lineage/README.md b/examples/lineage/README.md index 17846e9a3..fc289b530 100644 --- a/examples/lineage/README.md +++ b/examples/lineage/README.md @@ -1,3 +1,22 @@ + + # Lineage + Apache Hamilton ![Lineage Image](../../examples/lineage/lineage_v1.png) diff --git a/examples/lineage/data_loading.py b/examples/lineage/data_loading.py index 49390efd8..b053afee4 100644 --- a/examples/lineage/data_loading.py +++ b/examples/lineage/data_loading.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import List import pandas as pd diff --git a/examples/lineage/features.py b/examples/lineage/features.py index 1152da7b1..c185ea73a 100644 --- a/examples/lineage/features.py +++ b/examples/lineage/features.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import pandas as pd from sklearn import preprocessing diff --git a/examples/lineage/lineage_commands.py b/examples/lineage/lineage_commands.py index 74b16f6d3..849c32682 100644 --- a/examples/lineage/lineage_commands.py +++ b/examples/lineage/lineage_commands.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ The code is an example utility script that you could write to help you interface with Hamilton and its lineage capabilities. diff --git a/examples/lineage/lineage_script.py b/examples/lineage/lineage_script.py index 24c3d2355..9e3946147 100644 --- a/examples/lineage/lineage_script.py +++ b/examples/lineage/lineage_script.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """This is a script at linearly walks through some lineage functionality that Hamilton has. It mirrors the code that was presented for the Lineage + Hamilton in 10 minutes blog post. diff --git a/examples/lineage/lineage_snippets.ipynb b/examples/lineage/lineage_snippets.ipynb index 9e5947f6d..72b2c4190 100644 --- a/examples/lineage/lineage_snippets.ipynb +++ b/examples/lineage/lineage_snippets.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/lineage/model_pipeline.py b/examples/lineage/model_pipeline.py index 5fe71a3c6..e25c666c4 100644 --- a/examples/lineage/model_pipeline.py +++ b/examples/lineage/model_pipeline.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Dict, Union import pandas as pd diff --git a/examples/lineage/sets.py b/examples/lineage/sets.py index 2482b0330..480f5003a 100644 --- a/examples/lineage/sets.py +++ b/examples/lineage/sets.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from hamilton.function_modifiers import tag diff --git a/examples/make_python_virtualenv.sh b/examples/make_python_virtualenv.sh index d2dc23b85..127014b50 100644 --- a/examples/make_python_virtualenv.sh +++ b/examples/make_python_virtualenv.sh @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # This script is used to create separate python virtual environments for individual examples # A python virtual environment named "hamilton-env" is created in every directory containing requirements.txt file diff --git a/examples/materialization/README.md b/examples/materialization/README.md index a1d8684c4..206edd100 100644 --- a/examples/materialization/README.md +++ b/examples/materialization/README.md @@ -1,3 +1,22 @@ + + # Materialization Apache Hamilton's driver allows for ad-hoc materialization. This enables you to take a DAG you already have, diff --git a/examples/materialization/custom_materializers.py b/examples/materialization/custom_materializers.py index a2508ae43..95fe9da18 100644 --- a/examples/materialization/custom_materializers.py +++ b/examples/materialization/custom_materializers.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import dataclasses import pickle from typing import Any, Collection, Dict, Type diff --git a/examples/materialization/data_loaders.py b/examples/materialization/data_loaders.py index e31061eab..6cde1c4a3 100644 --- a/examples/materialization/data_loaders.py +++ b/examples/materialization/data_loaders.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np from sklearn import datasets, utils diff --git a/examples/materialization/datasaver_dataloader_example/README.md b/examples/materialization/datasaver_dataloader_example/README.md index ea163ab84..bff845555 100644 --- a/examples/materialization/datasaver_dataloader_example/README.md +++ b/examples/materialization/datasaver_dataloader_example/README.md @@ -1,3 +1,22 @@ + + # @datasaver and @dataloader example This example shows you can use the diff --git a/examples/materialization/datasaver_dataloader_example/notebook.ipynb b/examples/materialization/datasaver_dataloader_example/notebook.ipynb index 3d9eaaeb9..9f5ab77c9 100644 --- a/examples/materialization/datasaver_dataloader_example/notebook.ipynb +++ b/examples/materialization/datasaver_dataloader_example/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/materialization/datasaver_dataloader_example/run.py b/examples/materialization/datasaver_dataloader_example/run.py index bdedf98e0..3cfb2f100 100644 --- a/examples/materialization/datasaver_dataloader_example/run.py +++ b/examples/materialization/datasaver_dataloader_example/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import simple_etl from hamilton import driver diff --git a/examples/materialization/datasaver_dataloader_example/simple_etl.py b/examples/materialization/datasaver_dataloader_example/simple_etl.py index c7f02979c..d19584d0e 100644 --- a/examples/materialization/datasaver_dataloader_example/simple_etl.py +++ b/examples/materialization/datasaver_dataloader_example/simple_etl.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from sklearn import datasets diff --git a/examples/materialization/model_training.py b/examples/materialization/model_training.py index 894cc533a..2aa92f1e0 100644 --- a/examples/materialization/model_training.py +++ b/examples/materialization/model_training.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Dict import numpy as np diff --git a/examples/materialization/notebook.ipynb b/examples/materialization/notebook.ipynb index d1d117f09..951f88924 100644 --- a/examples/materialization/notebook.ipynb +++ b/examples/materialization/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/materialization/run.py b/examples/materialization/run.py index fce1faee7..30bf545eb 100644 --- a/examples/materialization/run.py +++ b/examples/materialization/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Example script showing how one might setup a generic model training pipeline that is quickly configurable. """ diff --git a/examples/mlflow/README.md b/examples/mlflow/README.md index 4c7e512bd..e49f7660f 100644 --- a/examples/mlflow/README.md +++ b/examples/mlflow/README.md @@ -1,3 +1,22 @@ + + # MLFLow plugin for Apache Hamilton [MLFlow](https://mlflow.org/) is an open-source Python framework for experiment tracking. It allows data science teams to store results, artifacts (machine learning models, figures, tables), and metadata in a principled way when executing data pipelines. diff --git a/examples/mlflow/tutorial.ipynb b/examples/mlflow/tutorial.ipynb index 0b06ddb04..db3c275f7 100644 --- a/examples/mlflow/tutorial.ipynb +++ b/examples/mlflow/tutorial.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/model_examples/README.md b/examples/model_examples/README.md index 3d1e5fa43..59c21ea6a 100644 --- a/examples/model_examples/README.md +++ b/examples/model_examples/README.md @@ -1,3 +1,22 @@ + + # Creating model training/predict workflows with Apache Hamilton Apache Hamilton is a general purpose dataflow framework. So you can diff --git a/examples/model_examples/modular_example/README.md b/examples/model_examples/modular_example/README.md index c0076c04b..46d4538f5 100644 --- a/examples/model_examples/modular_example/README.md +++ b/examples/model_examples/modular_example/README.md @@ -1,3 +1,22 @@ + + # Modular pipeline example In this example we show how you can compose a pipeline from multiple modules. diff --git a/examples/model_examples/modular_example/features.py b/examples/model_examples/modular_example/features.py index a18f7e84b..434f8931d 100644 --- a/examples/model_examples/modular_example/features.py +++ b/examples/model_examples/modular_example/features.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd diff --git a/examples/model_examples/modular_example/inference.py b/examples/model_examples/modular_example/inference.py index fdf79e73a..6fbdbbec5 100644 --- a/examples/model_examples/modular_example/inference.py +++ b/examples/model_examples/modular_example/inference.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Any import pandas as pd diff --git a/examples/model_examples/modular_example/notebook.ipynb b/examples/model_examples/modular_example/notebook.ipynb index c3c3eef07..8c913a4f7 100644 --- a/examples/model_examples/modular_example/notebook.ipynb +++ b/examples/model_examples/modular_example/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "metadata": {}, "cell_type": "code", diff --git a/examples/model_examples/modular_example/pipeline.py b/examples/model_examples/modular_example/pipeline.py index 0c7ebb0eb..7fc09caf6 100644 --- a/examples/model_examples/modular_example/pipeline.py +++ b/examples/model_examples/modular_example/pipeline.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Any import features diff --git a/examples/model_examples/modular_example/run.py b/examples/model_examples/modular_example/run.py index 415bd2863..d15c0d6d7 100644 --- a/examples/model_examples/modular_example/run.py +++ b/examples/model_examples/modular_example/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pipeline from hamilton import driver diff --git a/examples/model_examples/modular_example/train.py b/examples/model_examples/modular_example/train.py index 9d0d93bea..50caf6d79 100644 --- a/examples/model_examples/modular_example/train.py +++ b/examples/model_examples/modular_example/train.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Any import pandas as pd diff --git a/examples/model_examples/scikit-learn/Hamilton_for_ML_dataflows.ipynb b/examples/model_examples/scikit-learn/Hamilton_for_ML_dataflows.ipynb index 80ee07a76..081d8b3f7 100644 --- a/examples/model_examples/scikit-learn/Hamilton_for_ML_dataflows.ipynb +++ b/examples/model_examples/scikit-learn/Hamilton_for_ML_dataflows.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/model_examples/scikit-learn/README.md b/examples/model_examples/scikit-learn/README.md index 21e425326..ead8d390c 100644 --- a/examples/model_examples/scikit-learn/README.md +++ b/examples/model_examples/scikit-learn/README.md @@ -1,3 +1,22 @@ + + # Using Apache Hamilton for ML dataflows Here we have a simple example showing how you can diff --git a/examples/model_examples/scikit-learn/digit_loader.py b/examples/model_examples/scikit-learn/digit_loader.py index bbd640723..53cdfd9a5 100644 --- a/examples/model_examples/scikit-learn/digit_loader.py +++ b/examples/model_examples/scikit-learn/digit_loader.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np from sklearn import datasets, utils diff --git a/examples/model_examples/scikit-learn/iris_loader.py b/examples/model_examples/scikit-learn/iris_loader.py index e0ecb67e2..8688c7df9 100644 --- a/examples/model_examples/scikit-learn/iris_loader.py +++ b/examples/model_examples/scikit-learn/iris_loader.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np from sklearn import datasets, utils diff --git a/examples/model_examples/scikit-learn/my_train_evaluate_logic.py b/examples/model_examples/scikit-learn/my_train_evaluate_logic.py index 0c56cad3e..ccbdc69e8 100644 --- a/examples/model_examples/scikit-learn/my_train_evaluate_logic.py +++ b/examples/model_examples/scikit-learn/my_train_evaluate_logic.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Dict import numpy as np diff --git a/examples/model_examples/scikit-learn/run.py b/examples/model_examples/scikit-learn/run.py index d49ee5ba0..0928bb5ee 100644 --- a/examples/model_examples/scikit-learn/run.py +++ b/examples/model_examples/scikit-learn/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Example script showing how one might setup a generic model training pipeline that is quickly configurable. """ diff --git a/examples/model_examples/time-series/Hamilton-TimeSeriesmodel.ipynb b/examples/model_examples/time-series/Hamilton-TimeSeriesmodel.ipynb index 2214a641c..0adf25bd6 100644 --- a/examples/model_examples/time-series/Hamilton-TimeSeriesmodel.ipynb +++ b/examples/model_examples/time-series/Hamilton-TimeSeriesmodel.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/model_examples/time-series/README.md b/examples/model_examples/time-series/README.md index 3a91ed204..93412d65a 100644 --- a/examples/model_examples/time-series/README.md +++ b/examples/model_examples/time-series/README.md @@ -1,3 +1,22 @@ + + # Kaggle M5 Forecasting Source - https://www.kaggle.com/c/m5-forecasting-accuracy. diff --git a/examples/model_examples/time-series/data_loaders.py b/examples/model_examples/time-series/data_loaders.py index 6d0558fd7..8e2d4da8c 100644 --- a/examples/model_examples/time-series/data_loaders.py +++ b/examples/model_examples/time-series/data_loaders.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import gc import logging import os diff --git a/examples/model_examples/time-series/model_pipeline.py b/examples/model_examples/time-series/model_pipeline.py index bc7f0253b..1cdd63ed3 100644 --- a/examples/model_examples/time-series/model_pipeline.py +++ b/examples/model_examples/time-series/model_pipeline.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import gc import logging diff --git a/examples/model_examples/time-series/run.py b/examples/model_examples/time-series/run.py index 0c96d2686..64eb9e49e 100644 --- a/examples/model_examples/time-series/run.py +++ b/examples/model_examples/time-series/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging import sys import time diff --git a/examples/model_examples/time-series/transforms.py b/examples/model_examples/time-series/transforms.py index 4a9e8cac2..299a964a8 100644 --- a/examples/model_examples/time-series/transforms.py +++ b/examples/model_examples/time-series/transforms.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Tuple import pandas as pd diff --git a/examples/model_examples/time-series/utils.py b/examples/model_examples/time-series/utils.py index 5293d5e8c..854cd8713 100644 --- a/examples/model_examples/time-series/utils.py +++ b/examples/model_examples/time-series/utils.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging import numpy as np diff --git a/examples/module_overrides/README.md b/examples/module_overrides/README.md index 932ca07e6..14ca25f61 100644 --- a/examples/module_overrides/README.md +++ b/examples/module_overrides/README.md @@ -1,3 +1,22 @@ + + # Allowing modules to override same named functions When constructing the ``Driver``, we can import several modules: diff --git a/examples/module_overrides/module_a.py b/examples/module_overrides/module_a.py index b83d4001f..359922f2c 100644 --- a/examples/module_overrides/module_a.py +++ b/examples/module_overrides/module_a.py @@ -1,2 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + def foo() -> str: return "This is module a." diff --git a/examples/module_overrides/module_b.py b/examples/module_overrides/module_b.py index a5db01b81..3003ebee0 100644 --- a/examples/module_overrides/module_b.py +++ b/examples/module_overrides/module_b.py @@ -1,2 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + def foo() -> str: return "This is module b." diff --git a/examples/module_overrides/run.py b/examples/module_overrides/run.py index 20a8bf545..ec79ac5c6 100644 --- a/examples/module_overrides/run.py +++ b/examples/module_overrides/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import module_a import module_b diff --git a/examples/mutate/abstract_functionality_blueprint/README b/examples/mutate/abstract_functionality_blueprint/README index 342fc6033..012ec4e00 100644 --- a/examples/mutate/abstract_functionality_blueprint/README +++ b/examples/mutate/abstract_functionality_blueprint/README @@ -1,3 +1,22 @@ + + # Mutate We give some application suggestions for mutating the outputs of functions in a distributed manner with `@mutate.` diff --git a/examples/mutate/abstract_functionality_blueprint/mutate.py b/examples/mutate/abstract_functionality_blueprint/mutate.py index ae1d7326b..786e5982c 100644 --- a/examples/mutate/abstract_functionality_blueprint/mutate.py +++ b/examples/mutate/abstract_functionality_blueprint/mutate.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Any, List import pandas as pd diff --git a/examples/mutate/abstract_functionality_blueprint/mutate_on_output.py b/examples/mutate/abstract_functionality_blueprint/mutate_on_output.py index fb901adc3..571a44847 100644 --- a/examples/mutate/abstract_functionality_blueprint/mutate_on_output.py +++ b/examples/mutate/abstract_functionality_blueprint/mutate_on_output.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Any, Dict, List import pandas as pd diff --git a/examples/mutate/abstract_functionality_blueprint/mutate_twice_the_same.py b/examples/mutate/abstract_functionality_blueprint/mutate_twice_the_same.py index b67fa26a3..8ec0297dc 100644 --- a/examples/mutate/abstract_functionality_blueprint/mutate_twice_the_same.py +++ b/examples/mutate/abstract_functionality_blueprint/mutate_twice_the_same.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from hamilton.function_modifiers import mutate diff --git a/examples/mutate/abstract_functionality_blueprint/notebook.ipynb b/examples/mutate/abstract_functionality_blueprint/notebook.ipynb index f6061ea4c..a73c36ea7 100644 --- a/examples/mutate/abstract_functionality_blueprint/notebook.ipynb +++ b/examples/mutate/abstract_functionality_blueprint/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/mutate/abstract_functionality_blueprint/pipe_output.py b/examples/mutate/abstract_functionality_blueprint/pipe_output.py index f6c4fec38..47e8263e0 100644 --- a/examples/mutate/abstract_functionality_blueprint/pipe_output.py +++ b/examples/mutate/abstract_functionality_blueprint/pipe_output.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Any, List import pandas as pd diff --git a/examples/mutate/abstract_functionality_blueprint/pipe_output_on_output.py b/examples/mutate/abstract_functionality_blueprint/pipe_output_on_output.py index 7649778bf..a3ccdfac8 100644 --- a/examples/mutate/abstract_functionality_blueprint/pipe_output_on_output.py +++ b/examples/mutate/abstract_functionality_blueprint/pipe_output_on_output.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Dict from hamilton.function_modifiers import ( diff --git a/examples/mutate/abstract_functionality_blueprint/procedural.py b/examples/mutate/abstract_functionality_blueprint/procedural.py index f95d5a0a0..11b6b2ee0 100644 --- a/examples/mutate/abstract_functionality_blueprint/procedural.py +++ b/examples/mutate/abstract_functionality_blueprint/procedural.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Any, List import pandas as pd diff --git a/examples/narwhals/README.md b/examples/narwhals/README.md index 12e2d148e..d28901f29 100644 --- a/examples/narwhals/README.md +++ b/examples/narwhals/README.md @@ -1,3 +1,22 @@ + + # Narwhals [Narwhals](https://narwhals-dev.github.io/narwhals/) is a library that aims diff --git a/examples/narwhals/example.py b/examples/narwhals/example.py index 1b70310b3..c367d8765 100644 --- a/examples/narwhals/example.py +++ b/examples/narwhals/example.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import narwhals as nw import pandas as pd import polars as pl diff --git a/examples/narwhals/notebook.ipynb b/examples/narwhals/notebook.ipynb index a0fce5ab1..d7d7b0314 100644 --- a/examples/narwhals/notebook.ipynb +++ b/examples/narwhals/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/numpy/air-quality-analysis/README.md b/examples/numpy/air-quality-analysis/README.md index 4ac67336d..36215daea 100644 --- a/examples/numpy/air-quality-analysis/README.md +++ b/examples/numpy/air-quality-analysis/README.md @@ -1,3 +1,22 @@ + + # Air Quality Analysis This is taken from the numpy tutorial https://github.com/numpy/numpy-tutorials/blob/main/content/tutorial-air-quality-analysis.md. diff --git a/examples/numpy/air-quality-analysis/analysis_flow.py b/examples/numpy/air-quality-analysis/analysis_flow.py index c3b37c82a..1830db962 100644 --- a/examples/numpy/air-quality-analysis/analysis_flow.py +++ b/examples/numpy/air-quality-analysis/analysis_flow.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This code is based on the example presented in https://github.com/numpy/numpy-tutorials/blob/main/content/tutorial-air-quality-analysis.md diff --git a/examples/numpy/air-quality-analysis/hamilton_notebook.ipynb b/examples/numpy/air-quality-analysis/hamilton_notebook.ipynb index c75343bd9..86fbe5d94 100644 --- a/examples/numpy/air-quality-analysis/hamilton_notebook.ipynb +++ b/examples/numpy/air-quality-analysis/hamilton_notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/numpy/air-quality-analysis/run_analysis.py b/examples/numpy/air-quality-analysis/run_analysis.py index 754ae30a3..bfbd87739 100644 --- a/examples/numpy/air-quality-analysis/run_analysis.py +++ b/examples/numpy/air-quality-analysis/run_analysis.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import analysis_flow from hamilton import base, driver diff --git a/examples/openlineage/README.md b/examples/openlineage/README.md index 86cdc1458..973f49c34 100644 --- a/examples/openlineage/README.md +++ b/examples/openlineage/README.md @@ -1,3 +1,22 @@ + + # OpenLineage Adapter This is an example of how to use the OpenLineage adapter that can be used to send metadata to an OpenLineage server. diff --git a/examples/openlineage/fake_data.py b/examples/openlineage/fake_data.py index 02450ad9f..85f67e687 100644 --- a/examples/openlineage/fake_data.py +++ b/examples/openlineage/fake_data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Module to help generate fake data for testing purposes. """ diff --git a/examples/openlineage/notebook.ipynb b/examples/openlineage/notebook.ipynb index 3a9b95987..cababdcb4 100644 --- a/examples/openlineage/notebook.ipynb +++ b/examples/openlineage/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/openlineage/pipeline.py b/examples/openlineage/pipeline.py index f9b0c74b8..1178d3d5f 100644 --- a/examples/openlineage/pipeline.py +++ b/examples/openlineage/pipeline.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pickle from typing import Tuple diff --git a/examples/openlineage/run.py b/examples/openlineage/run.py index 6dcdf3e5e..deda4b147 100644 --- a/examples/openlineage/run.py +++ b/examples/openlineage/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import sqlite3 import pipeline diff --git a/examples/opentelemetry/README.md b/examples/opentelemetry/README.md index e98aad78f..53da2d44c 100644 --- a/examples/opentelemetry/README.md +++ b/examples/opentelemetry/README.md @@ -1,3 +1,22 @@ + + # Apache Hamilton + OpenTelemetry You can use the `hamilton.plugins.OpenTelemetryTracer()` to send telemetry diff --git a/examples/pandas/materialization/my_functions.py b/examples/pandas/materialization/my_functions.py index b54f6b171..c57618842 100644 --- a/examples/pandas/materialization/my_functions.py +++ b/examples/pandas/materialization/my_functions.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd """ diff --git a/examples/pandas/materialization/my_script.py b/examples/pandas/materialization/my_script.py index 252def815..d42fb192b 100644 --- a/examples/pandas/materialization/my_script.py +++ b/examples/pandas/materialization/my_script.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging import sqlite3 import sys diff --git a/examples/pandas/materialization/notebook.ipynb b/examples/pandas/materialization/notebook.ipynb index 601115b24..73a2677ee 100644 --- a/examples/pandas/materialization/notebook.ipynb +++ b/examples/pandas/materialization/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/pandas/split-apply-combine/README.md b/examples/pandas/split-apply-combine/README.md index 5ecca18cf..733679be4 100644 --- a/examples/pandas/split-apply-combine/README.md +++ b/examples/pandas/split-apply-combine/README.md @@ -1,3 +1,22 @@ + + # Split / Apply / Combine This example demonstrates how to perform diff --git a/examples/pandas/split-apply-combine/my_functions.py b/examples/pandas/split-apply-combine/my_functions.py index 218dda155..dda9b3bed 100644 --- a/examples/pandas/split-apply-combine/my_functions.py +++ b/examples/pandas/split-apply-combine/my_functions.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Dict import numpy as np diff --git a/examples/pandas/split-apply-combine/my_script.py b/examples/pandas/split-apply-combine/my_script.py index d1ca28297..5c3c49143 100644 --- a/examples/pandas/split-apply-combine/my_script.py +++ b/examples/pandas/split-apply-combine/my_script.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from inspect import cleandoc from io import StringIO diff --git a/examples/pandas/split-apply-combine/my_wrapper.py b/examples/pandas/split-apply-combine/my_wrapper.py index 95ee04dde..77c210aca 100644 --- a/examples/pandas/split-apply-combine/my_wrapper.py +++ b/examples/pandas/split-apply-combine/my_wrapper.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Dict import my_functions diff --git a/examples/pandas/split-apply-combine/notebook.ipynb b/examples/pandas/split-apply-combine/notebook.ipynb index 94b134d95..26af76012 100644 --- a/examples/pandas/split-apply-combine/notebook.ipynb +++ b/examples/pandas/split-apply-combine/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/pandas/with_columns/README b/examples/pandas/with_columns/README index 53a422d5a..78eec3c60 100644 --- a/examples/pandas/with_columns/README +++ b/examples/pandas/with_columns/README @@ -1,3 +1,22 @@ + + # Using with_columns with Pandas We show the ability to use the familiar `with_columns` from either `pyspark` or `polars` on a Pandas dataframe. diff --git a/examples/pandas/with_columns/my_functions.py b/examples/pandas/with_columns/my_functions.py index 5ce250227..35093b8bb 100644 --- a/examples/pandas/with_columns/my_functions.py +++ b/examples/pandas/with_columns/my_functions.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from hamilton.function_modifiers import config diff --git a/examples/pandas/with_columns/notebook.ipynb b/examples/pandas/with_columns/notebook.ipynb index 94953a288..affae9070 100644 --- a/examples/pandas/with_columns/notebook.ipynb +++ b/examples/pandas/with_columns/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/parallelism/README.md b/examples/parallelism/README.md index 02a6426f0..c8f47ba05 100644 --- a/examples/parallelism/README.md +++ b/examples/parallelism/README.md @@ -1,3 +1,22 @@ + + Here you'll find some examples that demonstrate how to use Apache Hamilton's parallelism features. Note: serialization caveats still apply. For more information see the diff --git a/examples/parallelism/file_processing/README.md b/examples/parallelism/file_processing/README.md index 9d412c4f3..dc3ae9455 100644 --- a/examples/parallelism/file_processing/README.md +++ b/examples/parallelism/file_processing/README.md @@ -1,3 +1,22 @@ + + # Parallelism Example ## Overview diff --git a/examples/parallelism/file_processing/aggregate_data.py b/examples/parallelism/file_processing/aggregate_data.py index 952a17d29..0c4075e85 100644 --- a/examples/parallelism/file_processing/aggregate_data.py +++ b/examples/parallelism/file_processing/aggregate_data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from hamilton.htypes import Collect diff --git a/examples/parallelism/file_processing/list_data.py b/examples/parallelism/file_processing/list_data.py index 00b99416f..11da3307b 100644 --- a/examples/parallelism/file_processing/list_data.py +++ b/examples/parallelism/file_processing/list_data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import dataclasses import os from typing import List diff --git a/examples/parallelism/file_processing/notebook.ipynb b/examples/parallelism/file_processing/notebook.ipynb index cd30a4e30..3d3025e2f 100644 --- a/examples/parallelism/file_processing/notebook.ipynb +++ b/examples/parallelism/file_processing/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/parallelism/file_processing/process_data.py b/examples/parallelism/file_processing/process_data.py index 8a6e36f94..b697c1a86 100644 --- a/examples/parallelism/file_processing/process_data.py +++ b/examples/parallelism/file_processing/process_data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from list_data import CityData diff --git a/examples/parallelism/file_processing/run.py b/examples/parallelism/file_processing/run.py index d21f02dc2..e2f378325 100644 --- a/examples/parallelism/file_processing/run.py +++ b/examples/parallelism/file_processing/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging import aggregate_data diff --git a/examples/parallelism/graceful_running/README.md b/examples/parallelism/graceful_running/README.md index 635f89653..5215433a6 100644 --- a/examples/parallelism/graceful_running/README.md +++ b/examples/parallelism/graceful_running/README.md @@ -1,3 +1,22 @@ + + # Parallelism with GracefulErrorAdapter Example ## Overview diff --git a/examples/parallelism/graceful_running/functions.py b/examples/parallelism/graceful_running/functions.py index 07507ad53..9482ccce2 100644 --- a/examples/parallelism/graceful_running/functions.py +++ b/examples/parallelism/graceful_running/functions.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from itertools import cycle from typing import Any, Callable, Iterable, List, Tuple, Union diff --git a/examples/parallelism/graceful_running/run.py b/examples/parallelism/graceful_running/run.py index fd19881ce..43e747070 100644 --- a/examples/parallelism/graceful_running/run.py +++ b/examples/parallelism/graceful_running/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Iterable, Tuple import click diff --git a/examples/parallelism/lazy_threadpool_execution/README.md b/examples/parallelism/lazy_threadpool_execution/README.md index e374a9570..38a79db76 100644 --- a/examples/parallelism/lazy_threadpool_execution/README.md +++ b/examples/parallelism/lazy_threadpool_execution/README.md @@ -1,3 +1,22 @@ + + # Lazy threadpool execution [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/dagworks-inc/hamilton/blob/main/examples/parallelism/lazy_threadpool_execution/notebook.ipynb) diff --git a/examples/parallelism/lazy_threadpool_execution/my_functions.py b/examples/parallelism/lazy_threadpool_execution/my_functions.py index 59e9b9762..e8119072e 100644 --- a/examples/parallelism/lazy_threadpool_execution/my_functions.py +++ b/examples/parallelism/lazy_threadpool_execution/my_functions.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import time diff --git a/examples/parallelism/lazy_threadpool_execution/my_functions_async.py b/examples/parallelism/lazy_threadpool_execution/my_functions_async.py index a639ed963..8372b9838 100644 --- a/examples/parallelism/lazy_threadpool_execution/my_functions_async.py +++ b/examples/parallelism/lazy_threadpool_execution/my_functions_async.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import asyncio diff --git a/examples/parallelism/lazy_threadpool_execution/notebook.ipynb b/examples/parallelism/lazy_threadpool_execution/notebook.ipynb index a1253012c..a0e5f8f7c 100644 --- a/examples/parallelism/lazy_threadpool_execution/notebook.ipynb +++ b/examples/parallelism/lazy_threadpool_execution/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/parallelism/lazy_threadpool_execution/run.py b/examples/parallelism/lazy_threadpool_execution/run.py index 734842c39..3761be20c 100644 --- a/examples/parallelism/lazy_threadpool_execution/run.py +++ b/examples/parallelism/lazy_threadpool_execution/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import time import my_functions diff --git a/examples/parallelism/star_counting/README.md b/examples/parallelism/star_counting/README.md index f307238b6..496cdf904 100644 --- a/examples/parallelism/star_counting/README.md +++ b/examples/parallelism/star_counting/README.md @@ -1,3 +1,22 @@ + + # Star counting with Apache Hamilton/Parallelism This example goes along with the "Counting Stars with Apache Hamilton" blog post. diff --git a/examples/parallelism/star_counting/functions.py b/examples/parallelism/star_counting/functions.py index 5d5c74272..4118a6f1b 100644 --- a/examples/parallelism/star_counting/functions.py +++ b/examples/parallelism/star_counting/functions.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from datetime import datetime from typing import Dict, List, Tuple diff --git a/examples/parallelism/star_counting/notebook.ipynb b/examples/parallelism/star_counting/notebook.ipynb index 65f1a07ae..a7f798ec7 100644 --- a/examples/parallelism/star_counting/notebook.ipynb +++ b/examples/parallelism/star_counting/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/parallelism/star_counting/run.py b/examples/parallelism/star_counting/run.py index 7dd5f3844..8b6cc3b12 100644 --- a/examples/parallelism/star_counting/run.py +++ b/examples/parallelism/star_counting/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging from typing import Tuple diff --git a/examples/people_data_labs/README.md b/examples/people_data_labs/README.md index 02896ed4a..e2539b3c9 100644 --- a/examples/people_data_labs/README.md +++ b/examples/people_data_labs/README.md @@ -1,3 +1,22 @@ + + # People Data Labs [People Data Labs](https://www.peopledatalabs.com/) is a data provider that offers several data APIs for person enrichment & search, company enrichment & search, and IP enrichment. diff --git a/examples/people_data_labs/analysis.py b/examples/people_data_labs/analysis.py index ef527d643..57310a0ea 100644 --- a/examples/people_data_labs/analysis.py +++ b/examples/people_data_labs/analysis.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from pathlib import Path import pandas as pd diff --git a/examples/people_data_labs/download_data.py b/examples/people_data_labs/download_data.py index 5e4008a59..05d186e63 100644 --- a/examples/people_data_labs/download_data.py +++ b/examples/people_data_labs/download_data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import boto3 diff --git a/examples/people_data_labs/notebook.ipynb b/examples/people_data_labs/notebook.ipynb index 113d10ba0..20f75f83c 100644 --- a/examples/people_data_labs/notebook.ipynb +++ b/examples/people_data_labs/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/people_data_labs/run.py b/examples/people_data_labs/run.py index 8edae24fd..77925abb9 100644 --- a/examples/people_data_labs/run.py +++ b/examples/people_data_labs/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import analysis from hamilton import driver diff --git a/examples/plotly/README.md b/examples/plotly/README.md index 941db33ef..5efc3c929 100644 --- a/examples/plotly/README.md +++ b/examples/plotly/README.md @@ -1,3 +1,22 @@ + + # Plotly materializer extension By importing `hamilton.plugins.plotly_extensions`, you can register two additional materializers for Plotly figures. The `to.plotly()` creates static image files ([docs](https://plotly.com/python/static-image-export/)) and the `to.html()` outputs interactive HTML files ([docs](https://plotly.com/python/interactive-html-export/)). diff --git a/examples/plotly/model_training.py b/examples/plotly/model_training.py index 5d148c9df..29448f1e2 100644 --- a/examples/plotly/model_training.py +++ b/examples/plotly/model_training.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Dict import numpy as np diff --git a/examples/plotly/notebook.ipynb b/examples/plotly/notebook.ipynb index 8d833727f..62d16abdd 100644 --- a/examples/plotly/notebook.ipynb +++ b/examples/plotly/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/polars/README.md b/examples/polars/README.md index 45c9aaabc..8ae20344d 100644 --- a/examples/polars/README.md +++ b/examples/polars/README.md @@ -1,3 +1,22 @@ + + # Classic Apache Hamilton Hello World In this example we show you how to create a simple hello world dataflow that diff --git a/examples/polars/lazyframe/README.md b/examples/polars/lazyframe/README.md index 23dbe5fa3..211d6f111 100644 --- a/examples/polars/lazyframe/README.md +++ b/examples/polars/lazyframe/README.md @@ -1,3 +1,22 @@ + + # Classic Apache Hamilton Hello World In this example we show you how to create a simple hello world dataflow that diff --git a/examples/polars/lazyframe/my_functions.py b/examples/polars/lazyframe/my_functions.py index cc1d6ef62..92e0153f0 100644 --- a/examples/polars/lazyframe/my_functions.py +++ b/examples/polars/lazyframe/my_functions.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import polars as pl from hamilton.function_modifiers import load_from, value diff --git a/examples/polars/lazyframe/my_script.py b/examples/polars/lazyframe/my_script.py index 0e0fe4ad9..3c51cdbc2 100644 --- a/examples/polars/lazyframe/my_script.py +++ b/examples/polars/lazyframe/my_script.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging import sys diff --git a/examples/polars/materialization/README.md b/examples/polars/materialization/README.md index fc3b8407b..a2f2b882f 100644 --- a/examples/polars/materialization/README.md +++ b/examples/polars/materialization/README.md @@ -1,3 +1,22 @@ + + # Polars materializer examples Specifically this example shows how one can "materialize" diff --git a/examples/polars/materialization/my_functions.py b/examples/polars/materialization/my_functions.py index 7b627b666..332c5a471 100644 --- a/examples/polars/materialization/my_functions.py +++ b/examples/polars/materialization/my_functions.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import polars as pl """ diff --git a/examples/polars/materialization/my_script.py b/examples/polars/materialization/my_script.py index b532ebf26..01593dd7c 100644 --- a/examples/polars/materialization/my_script.py +++ b/examples/polars/materialization/my_script.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging import sys diff --git a/examples/polars/materialization/notebook.ipynb b/examples/polars/materialization/notebook.ipynb index cec55598e..1b3fe0924 100644 --- a/examples/polars/materialization/notebook.ipynb +++ b/examples/polars/materialization/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/polars/my_functions.py b/examples/polars/my_functions.py index 89a6d60c6..caae9162a 100644 --- a/examples/polars/my_functions.py +++ b/examples/polars/my_functions.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import polars as pl from hamilton.function_modifiers import extract_columns diff --git a/examples/polars/my_script.py b/examples/polars/my_script.py index 112078b60..aa206b4d9 100644 --- a/examples/polars/my_script.py +++ b/examples/polars/my_script.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging import sys diff --git a/examples/polars/notebook.ipynb b/examples/polars/notebook.ipynb index b36ec9e99..31645e9c6 100644 --- a/examples/polars/notebook.ipynb +++ b/examples/polars/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/polars/with_columns/README b/examples/polars/with_columns/README index 385ef5fb7..c36330ca6 100644 --- a/examples/polars/with_columns/README +++ b/examples/polars/with_columns/README @@ -1,3 +1,22 @@ + + # Using with_columns with Polars We show the ability to use the familiar `with_columns` from `polars`. Supported for both: `pl.DataFrame` and `pl.LazyFrame`. diff --git a/examples/polars/with_columns/my_functions.py b/examples/polars/with_columns/my_functions.py index 3b2c401b9..ba20558a1 100644 --- a/examples/polars/with_columns/my_functions.py +++ b/examples/polars/with_columns/my_functions.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import polars as pl from hamilton.function_modifiers import config diff --git a/examples/polars/with_columns/my_functions_lazy.py b/examples/polars/with_columns/my_functions_lazy.py index 4b65b2ac2..f4bba063c 100644 --- a/examples/polars/with_columns/my_functions_lazy.py +++ b/examples/polars/with_columns/my_functions_lazy.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import polars as pl from hamilton.function_modifiers import config diff --git a/examples/polars/with_columns/notebook.ipynb b/examples/polars/with_columns/notebook.ipynb index 75174e6de..d827aa31d 100644 --- a/examples/polars/with_columns/notebook.ipynb +++ b/examples/polars/with_columns/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/prefect/README.md b/examples/prefect/README.md index 5deb9f74c..b2f84d988 100644 --- a/examples/prefect/README.md +++ b/examples/prefect/README.md @@ -1,3 +1,22 @@ + + # Apache Hamilton + Prefect In this example, were going to show how to run a simple `data preprocessing -> model training -> model evaluation` workflow using Apache Hamilton within Prefect tasks. diff --git a/examples/prefect/evaluate_model.py b/examples/prefect/evaluate_model.py index 6664d82a4..ed148b51d 100644 --- a/examples/prefect/evaluate_model.py +++ b/examples/prefect/evaluate_model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import scipy.stats as stats from sklearn.metrics import get_scorer diff --git a/examples/prefect/hamilton_prefect_notebook.ipynb b/examples/prefect/hamilton_prefect_notebook.ipynb index d746927c1..da7a294bc 100644 --- a/examples/prefect/hamilton_prefect_notebook.ipynb +++ b/examples/prefect/hamilton_prefect_notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/prefect/prepare_data.py b/examples/prefect/prepare_data.py index f1646e4f0..d4dd2c0a4 100644 --- a/examples/prefect/prepare_data.py +++ b/examples/prefect/prepare_data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import List import numpy as np diff --git a/examples/prefect/run.py b/examples/prefect/run.py index b1a98ab5e..e6f1e907c 100644 --- a/examples/prefect/run.py +++ b/examples/prefect/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import evaluate_model import pandas as pd diff --git a/examples/prefect/train_model.py b/examples/prefect/train_model.py index 5d0fa22de..60a243164 100644 --- a/examples/prefect/train_model.py +++ b/examples/prefect/train_model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Dict, List import numpy as np diff --git a/examples/ray/README.md b/examples/ray/README.md index c043eedb9..71bc3d9a7 100644 --- a/examples/ray/README.md +++ b/examples/ray/README.md @@ -1,3 +1,22 @@ + + # Scaling Apache Hamilton on Ray [Ray](https://ray.io) is a general purpose framework that allows for parallel diff --git a/examples/ray/hello_world/README.md b/examples/ray/hello_world/README.md index e40c3262c..0539cf496 100644 --- a/examples/ray/hello_world/README.md +++ b/examples/ray/hello_world/README.md @@ -1,3 +1,22 @@ + + # Apache Hamilton on Ray Here we have a hello world example showing how you can diff --git a/examples/ray/hello_world/data_loaders.py b/examples/ray/hello_world/data_loaders.py index 5d1b42b60..6af0467a4 100644 --- a/examples/ray/hello_world/data_loaders.py +++ b/examples/ray/hello_world/data_loaders.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd diff --git a/examples/ray/hello_world/notebook.ipynb b/examples/ray/hello_world/notebook.ipynb index 01a4fcb50..72221c516 100644 --- a/examples/ray/hello_world/notebook.ipynb +++ b/examples/ray/hello_world/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/ray/hello_world/run.py b/examples/ray/hello_world/run.py index bc82b7cc1..17cbd9305 100644 --- a/examples/ray/hello_world/run.py +++ b/examples/ray/hello_world/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import importlib import ray diff --git a/examples/ray/ray_Hamilton_UI_tracking/README b/examples/ray/ray_Hamilton_UI_tracking/README index 94cd6f8fc..8a249e683 100644 --- a/examples/ray/ray_Hamilton_UI_tracking/README +++ b/examples/ray/ray_Hamilton_UI_tracking/README @@ -1,3 +1,22 @@ + + # Tracking telemetry in Hamilton UI for Ray clusters We show the ability to combine the [RayGraphAdapter](https://hamilton.apache.org/reference/graph-adapters/RayGraphAdapter/) and [HamiltonTracker](https://hamilton.apache.org/concepts/ui/) to run a dummy DAG. diff --git a/examples/ray/ray_Hamilton_UI_tracking/hamilton_notebook.ipynb b/examples/ray/ray_Hamilton_UI_tracking/hamilton_notebook.ipynb index b061ff3c6..dcb1cab6b 100644 --- a/examples/ray/ray_Hamilton_UI_tracking/hamilton_notebook.ipynb +++ b/examples/ray/ray_Hamilton_UI_tracking/hamilton_notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, @@ -29,8 +36,8 @@ "from hamilton_sdk.adapters import HamiltonTracker\n", "\n", "# Inputs required to track into correct project in the UI\n", - "project_id = 2\n", - "username = \"admin\"\n", + "project_id = 1\n", + "username = \"elijah\"\n", "\n", "tracker_ray = HamiltonTracker(\n", " project_id=project_id,\n", diff --git a/examples/ray/ray_Hamilton_UI_tracking/ray_lineage.py b/examples/ray/ray_Hamilton_UI_tracking/ray_lineage.py index d9d20c2ca..003bee0c2 100644 --- a/examples/ray/ray_Hamilton_UI_tracking/ray_lineage.py +++ b/examples/ray/ray_Hamilton_UI_tracking/ray_lineage.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import time diff --git a/examples/ray/ray_Hamilton_UI_tracking/run_lineage.py b/examples/ray/ray_Hamilton_UI_tracking/run_lineage.py index 6d135efd1..264f9f344 100644 --- a/examples/ray/ray_Hamilton_UI_tracking/run_lineage.py +++ b/examples/ray/ray_Hamilton_UI_tracking/run_lineage.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import click import ray_lineage @@ -7,7 +24,7 @@ @click.command() -@click.option("--username", required=True, type=str) +@click.option("--username", default="elijah", required=True, type=str) @click.option("--project_id", default=1, type=int) def run(project_id, username): try: diff --git a/examples/reusing_functions/README.md b/examples/reusing_functions/README.md index 360b711c0..cecbb7624 100644 --- a/examples/reusing_functions/README.md +++ b/examples/reusing_functions/README.md @@ -1,3 +1,22 @@ + + # subdag operator This README demonstrates the use of the subdag operator. diff --git a/examples/reusing_functions/main.py b/examples/reusing_functions/main.py index 6eae4d76b..d74a252d5 100644 --- a/examples/reusing_functions/main.py +++ b/examples/reusing_functions/main.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Any, Dict import pandas as pd diff --git a/examples/reusing_functions/reusable_subdags.py b/examples/reusing_functions/reusable_subdags.py index f50773b57..2dac40971 100644 --- a/examples/reusing_functions/reusable_subdags.py +++ b/examples/reusing_functions/reusable_subdags.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd import unique_users diff --git a/examples/reusing_functions/reusing_functions.ipynb b/examples/reusing_functions/reusing_functions.ipynb index 24c05424d..7d8b8a69f 100644 --- a/examples/reusing_functions/reusing_functions.ipynb +++ b/examples/reusing_functions/reusing_functions.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/reusing_functions/unique_users.py b/examples/reusing_functions/unique_users.py index fe1a05be7..7d5e3c4a5 100644 --- a/examples/reusing_functions/unique_users.py +++ b/examples/reusing_functions/unique_users.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd _grain_mapping = {"day": "D", "week": "W", "month": "M"} diff --git a/examples/reverse_etl/README.md b/examples/reverse_etl/README.md index 48d44c2fa..ab03c98e6 100644 --- a/examples/reverse_etl/README.md +++ b/examples/reverse_etl/README.md @@ -1,3 +1,22 @@ + + # Reverse ETL for your timesheets This example shows how to create a timesheet from time tracking data generated by the command line tool [Timewarrior](https://timewarrior.net/) and upload it to Google Sheets. diff --git a/examples/reverse_etl/notebook.ipynb b/examples/reverse_etl/notebook.ipynb index e5c48810b..fe2d89a5b 100644 --- a/examples/reverse_etl/notebook.ipynb +++ b/examples/reverse_etl/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/reverse_etl/upload_timesheet.py b/examples/reverse_etl/upload_timesheet.py index 42c3c55b8..340d347c5 100644 --- a/examples/reverse_etl/upload_timesheet.py +++ b/examples/reverse_etl/upload_timesheet.py @@ -1,4 +1,21 @@ #!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import json import pathlib import sys diff --git a/examples/schema/README.md b/examples/schema/README.md index f3303e8cc..0762de1b7 100644 --- a/examples/schema/README.md +++ b/examples/schema/README.md @@ -1,3 +1,22 @@ + + # Schema tracking and validation This examples showcases the `SchemaValidator()` adapter. At runtime, it will collect the schema of DataFrame-like objects and stored them as `pyarrow.Schema` objects. On subsequent runs, it will check if the schema of a noed output matches the stored reference schema. diff --git a/examples/schema/dataflow.py b/examples/schema/dataflow.py index 9eb28d01b..8329eeb31 100644 --- a/examples/schema/dataflow.py +++ b/examples/schema/dataflow.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd diff --git a/examples/schema/multi_dataflow.py b/examples/schema/multi_dataflow.py index 62da6efb7..cc307191a 100644 --- a/examples/schema/multi_dataflow.py +++ b/examples/schema/multi_dataflow.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import duckdb import ibis import ibis.expr.types as ir diff --git a/examples/scikit-learn/species_distribution_modeling/README.md b/examples/scikit-learn/species_distribution_modeling/README.md index 62be47b81..c7e68e6f3 100644 --- a/examples/scikit-learn/species_distribution_modeling/README.md +++ b/examples/scikit-learn/species_distribution_modeling/README.md @@ -1,3 +1,22 @@ + + # Species distribution modeling We translate the Species distribution modeling from scikit-learn into Apache Hamilton to showcase `pipe` and`pipe_output`. diff --git a/examples/scikit-learn/species_distribution_modeling/grids.py b/examples/scikit-learn/species_distribution_modeling/grids.py index cad14858b..6c1ebe013 100644 --- a/examples/scikit-learn/species_distribution_modeling/grids.py +++ b/examples/scikit-learn/species_distribution_modeling/grids.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Tuple import numpy as np diff --git a/examples/scikit-learn/species_distribution_modeling/hamilton_notebook.ipynb b/examples/scikit-learn/species_distribution_modeling/hamilton_notebook.ipynb index 8a2d37f33..1e6656a9f 100644 --- a/examples/scikit-learn/species_distribution_modeling/hamilton_notebook.ipynb +++ b/examples/scikit-learn/species_distribution_modeling/hamilton_notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/scikit-learn/species_distribution_modeling/load_data.py b/examples/scikit-learn/species_distribution_modeling/load_data.py index 4e142cfd2..00da84035 100644 --- a/examples/scikit-learn/species_distribution_modeling/load_data.py +++ b/examples/scikit-learn/species_distribution_modeling/load_data.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from sklearn.datasets import fetch_species_distributions from sklearn.utils._bunch import Bunch diff --git a/examples/scikit-learn/species_distribution_modeling/original_script.py b/examples/scikit-learn/species_distribution_modeling/original_script.py index 9fb62b740..5e8d0662f 100644 --- a/examples/scikit-learn/species_distribution_modeling/original_script.py +++ b/examples/scikit-learn/species_distribution_modeling/original_script.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Authors: Peter Prettenhofer # Jake Vanderplas # diff --git a/examples/scikit-learn/species_distribution_modeling/postprocessing_results.py b/examples/scikit-learn/species_distribution_modeling/postprocessing_results.py index 4fab10686..a4fce770c 100644 --- a/examples/scikit-learn/species_distribution_modeling/postprocessing_results.py +++ b/examples/scikit-learn/species_distribution_modeling/postprocessing_results.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Any, Dict, Tuple import numpy as np diff --git a/examples/scikit-learn/species_distribution_modeling/preprocessing.py b/examples/scikit-learn/species_distribution_modeling/preprocessing.py index 5302364d0..083e8b27a 100644 --- a/examples/scikit-learn/species_distribution_modeling/preprocessing.py +++ b/examples/scikit-learn/species_distribution_modeling/preprocessing.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Dict, Tuple import numpy as np diff --git a/examples/scikit-learn/species_distribution_modeling/run.py b/examples/scikit-learn/species_distribution_modeling/run.py index 6cd737372..bc8f5a53e 100644 --- a/examples/scikit-learn/species_distribution_modeling/run.py +++ b/examples/scikit-learn/species_distribution_modeling/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import matplotlib.pyplot as plt diff --git a/examples/scikit-learn/species_distribution_modeling/train_and_predict.py b/examples/scikit-learn/species_distribution_modeling/train_and_predict.py index cda35a2bf..ea3a78c21 100644 --- a/examples/scikit-learn/species_distribution_modeling/train_and_predict.py +++ b/examples/scikit-learn/species_distribution_modeling/train_and_predict.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import numpy.typing as npt from sklearn import svm diff --git a/examples/scikit-learn/species_distribution_modeling/train_and_predict_using_mutate.py b/examples/scikit-learn/species_distribution_modeling/train_and_predict_using_mutate.py index 8e823c025..d9dd06f8a 100644 --- a/examples/scikit-learn/species_distribution_modeling/train_and_predict_using_mutate.py +++ b/examples/scikit-learn/species_distribution_modeling/train_and_predict_using_mutate.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import numpy.typing as npt from sklearn import svm diff --git a/examples/scikit-learn/transformer/README.md b/examples/scikit-learn/transformer/README.md index 1fe1eb696..1483af5ce 100644 --- a/examples/scikit-learn/transformer/README.md +++ b/examples/scikit-learn/transformer/README.md @@ -1,3 +1,22 @@ + + # Classic Apache Hamilton Hello World In this example we show you a custom scikit-learn `Transformer` class. This class should be compliant with [scikit-learn transformers specifications](https://scikit-learn.org/stable/developers/develop.html). This class is meant to be used as part of broader scikit-learn pipelines. Scikit-learn estimators and pipelines allow for stateful objects, which are helpful when applying transformations on train-test splits notably. Also, all pipeline, estimator, and transformer objects should be picklable, enabling reproducible pipelines. diff --git a/examples/scikit-learn/transformer/hamilton_notebook.ipynb b/examples/scikit-learn/transformer/hamilton_notebook.ipynb index 29922e904..76036e0e7 100644 --- a/examples/scikit-learn/transformer/hamilton_notebook.ipynb +++ b/examples/scikit-learn/transformer/hamilton_notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/scikit-learn/transformer/my_functions_a.py b/examples/scikit-learn/transformer/my_functions_a.py index 3da0e8333..3937c657a 100644 --- a/examples/scikit-learn/transformer/my_functions_a.py +++ b/examples/scikit-learn/transformer/my_functions_a.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd diff --git a/examples/scikit-learn/transformer/my_functions_b.py b/examples/scikit-learn/transformer/my_functions_b.py index 1fde4288d..4babc9429 100644 --- a/examples/scikit-learn/transformer/my_functions_b.py +++ b/examples/scikit-learn/transformer/my_functions_b.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd diff --git a/examples/scikit-learn/transformer/run.py b/examples/scikit-learn/transformer/run.py index 2df045a5b..ed131037e 100644 --- a/examples/scikit-learn/transformer/run.py +++ b/examples/scikit-learn/transformer/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import annotations import importlib diff --git a/examples/slack/README.md b/examples/slack/README.md index f6e9b3ead..8e1880e19 100644 --- a/examples/slack/README.md +++ b/examples/slack/README.md @@ -1,3 +1,22 @@ + + # Slack notifier example This example demonstrates how to use the Slack notifier. diff --git a/examples/slack/notebook.ipynb b/examples/slack/notebook.ipynb index ff4d26199..edf26e766 100644 --- a/examples/slack/notebook.ipynb +++ b/examples/slack/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/slack/slack_notification_example.py b/examples/slack/slack_notification_example.py index 29a1c3005..d34884d5f 100644 --- a/examples/slack/slack_notification_example.py +++ b/examples/slack/slack_notification_example.py @@ -1,23 +1,40 @@ -import pandas as pd - -from hamilton import driver -from hamilton.plugins.h_slack import SlackNotifier - - -def test_function() -> pd.DataFrame: - raise Exception("test exception") - return pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) - - -if __name__ == "__main__": - import __main__ - - api_key = "YOUR_API_KEY" - channel = "YOUR_CHANNEL" - dr = ( - driver.Builder() - .with_modules(__main__) - .with_adapters(SlackNotifier(api_key=api_key, channel=channel)) - .build() - ) - print(dr.execute(["test_function"])) +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pandas as pd + +from hamilton import driver +from hamilton.plugins.h_slack import SlackNotifier + + +def test_function() -> pd.DataFrame: + raise Exception("test exception") + return pd.DataFrame({"a": [1, 2, 3], "b": [4, 5, 6]}) + + +if __name__ == "__main__": + import __main__ + + api_key = "YOUR_API_KEY" + channel = "YOUR_CHANNEL" + dr = ( + driver.Builder() + .with_modules(__main__) + .with_adapters(SlackNotifier(api_key=api_key, channel=channel)) + .build() + ) + print(dr.execute(["test_function"])) diff --git a/examples/snowflake/hamilton_ui/Dockerfile b/examples/snowflake/hamilton_ui/Dockerfile index 0433ae7a8..a71115660 100644 --- a/examples/snowflake/hamilton_ui/Dockerfile +++ b/examples/snowflake/hamilton_ui/Dockerfile @@ -1,14 +1,31 @@ -FROM python:3.13.1-slim - -RUN apt update -RUN apt upgrade sqlite3 -y -RUN pip install "sf-hamilton[ui,sdk]" -RUN pip install flask - -ENV HAMILTON_ALLOWED_HOSTS=".snowflakecomputing.app" -ENV SERVER_PORT=8001 - -COPY pipeline_endpoint.py /pipeline_endpoint.py -COPY my_functions.py /my_functions.py - -ENTRYPOINT /bin/bash -c "(hamilton ui --base-dir /hamilton-basedir &) && python /pipeline_endpoint.py" +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM python:3.13.1-slim + +RUN apt update +RUN apt upgrade sqlite3 -y +RUN pip install "sf-hamilton[ui,sdk]" +RUN pip install flask + +ENV HAMILTON_ALLOWED_HOSTS=".snowflakecomputing.app" +ENV SERVER_PORT=8001 + +COPY pipeline_endpoint.py /pipeline_endpoint.py +COPY my_functions.py /my_functions.py + +ENTRYPOINT /bin/bash -c "(hamilton ui --base-dir /hamilton-basedir &) && python /pipeline_endpoint.py" diff --git a/examples/snowflake/hamilton_ui/README.md b/examples/snowflake/hamilton_ui/README.md index 05d58a9f5..b810b14f7 100644 --- a/examples/snowflake/hamilton_ui/README.md +++ b/examples/snowflake/hamilton_ui/README.md @@ -1,3 +1,22 @@ + + # Running the Apache Hamilton & the Apache Hamilton UI in Snowflake This example is code for the ["Observability of Python code and application logic with Apache Hamilton UI on Snowflake Container Services" post](https://medium.com/@pkantyka/observability-of-python-code-and-application-logic-with-hamilton-ui-on-snowflake-container-services-a26693b46635) by diff --git a/examples/snowflake/hamilton_ui/my_functions.py b/examples/snowflake/hamilton_ui/my_functions.py index 387d0b9fc..4babc9429 100644 --- a/examples/snowflake/hamilton_ui/my_functions.py +++ b/examples/snowflake/hamilton_ui/my_functions.py @@ -1,21 +1,38 @@ -import pandas as pd - - -def spend_mean(spend: pd.Series) -> float: - """Shows function creating a scalar. In this case it computes the mean of the entire column.""" - return spend.mean() - - -def spend_zero_mean(spend: pd.Series, spend_mean: float) -> pd.Series: - """Shows function that takes a scalar. In this case to zero mean spend.""" - return spend - spend_mean - - -def spend_std_dev(spend: pd.Series) -> float: - """Function that computes the standard deviation of the spend column.""" - return spend.std() - - -def spend_zero_mean_unit_variance(spend_zero_mean: pd.Series, spend_std_dev: float) -> pd.Series: - """Function showing one way to make spend have zero mean and unit variance.""" - return spend_zero_mean / spend_std_dev +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pandas as pd + + +def spend_mean(spend: pd.Series) -> float: + """Shows function creating a scalar. In this case it computes the mean of the entire column.""" + return spend.mean() + + +def spend_zero_mean(spend: pd.Series, spend_mean: float) -> pd.Series: + """Shows function that takes a scalar. In this case to zero mean spend.""" + return spend - spend_mean + + +def spend_std_dev(spend: pd.Series) -> float: + """Function that computes the standard deviation of the spend column.""" + return spend.std() + + +def spend_zero_mean_unit_variance(spend_zero_mean: pd.Series, spend_std_dev: float) -> pd.Series: + """Function showing one way to make spend have zero mean and unit variance.""" + return spend_zero_mean / spend_std_dev diff --git a/examples/snowflake/hamilton_ui/pipeline_endpoint.py b/examples/snowflake/hamilton_ui/pipeline_endpoint.py index 217077f0b..9acaeff97 100644 --- a/examples/snowflake/hamilton_ui/pipeline_endpoint.py +++ b/examples/snowflake/hamilton_ui/pipeline_endpoint.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ This module: - Defines a flask app that listens for POST requests on /echo diff --git a/examples/snowflake/hamilton_ui/snowflake.sql b/examples/snowflake/hamilton_ui/snowflake.sql index ba1afc159..3520c8d3f 100644 --- a/examples/snowflake/hamilton_ui/snowflake.sql +++ b/examples/snowflake/hamilton_ui/snowflake.sql @@ -1,103 +1,120 @@ --- For more details visit: --- https://medium.com/@pkantyka/observability-of-python-code-and-application-logic-with-hamilton-ui-on-snowflake-container-services-a26693b46635 - -CREATE OR REPLACE IMAGE REPOSITORY images; --- then docker build -t /snowflake-hamilton-ui . --- docker login -u --- docker push /snowflake-hamilton-ui -SHOW IMAGES IN IMAGE REPOSITORY ; - -CREATE STAGE hamilton_base ENCRYPTION = (type = 'SNOWFLAKE_SSE'); - - -CREATE SERVICE public.hamilton_ui -IN COMPUTE POOL TEST_POOL -FROM SPECIFICATION $$ -spec: - containers: - - name: hamiltonui - image: ////snowflake-hamilton-ui - volumeMounts: - - name: hamilton-basedir - mountPath: /hamilton-basedir - endpoints: - - name: entrypoint - port: 8001 - - name: hamilton - port: 8241 - public: true - volumes: - - name: hamilton-basedir - source: "@..hamilton_base" - $$ -QUERY_WAREHOUSE = -; - -CALL SYSTEM$GET_SERVICE_STATUS('..hamilton_ui'); - -CALL SYSTEM$GET_SERVICE_LOGS('..hamilton_ui', '0', 'hammiltonui', 1000); - -SHOW ENDPOINTS IN SERVICE public.hamilton_ui; - -CREATE OR REPLACE FUNCTION public.hamilton_pipeline (prj_id number, signups variant, spend variant, output_columns variant) - RETURNS VARIANT - SERVICE=public.hamilton_ui - ENDPOINT=entrypoint - AS '/echo'; - - -SELECT - public.hamilton_pipeline ( - 1, - [1, 10, 50, 100, 200, 400], - [10, 10, 20, 40, 40, 50], - [ 'spend', 'signups', 'spend_std_dev', 'spend_zero_mean_unit_variance' ] - ) as data; - -WITH input_data AS ( - SELECT - public.hamilton_pipeline ( - 1, - [1, 10, 50, 100, 200, 400], - [10, 10, 20, 40, 40, 50], - [ 'spend', 'signups', 'spend_std_dev', 'spend_zero_mean_unit_variance' ] - ) as data -), -flattened AS ( - SELECT - key AS metric_key, - value AS metric_value - FROM - input_data - left join LATERAL FLATTEN(input_data.data) -) -SELECT - * -FROM - flattened f; - -WITH input_data AS ( - SELECT - public.hamilton_pipeline ( - 1, - [1, 10, 50, 100, 200, 400], - [10, 10, 20, 40, 40, 50], - [ 'spend', 'signups', 'spend_std_dev', 'spend_zero_mean_unit_variance' ] - ) as data -), -flattened AS ( - SELECT - key AS metric_key, - value AS metric_value - FROM - input_data - left join LATERAL FLATTEN(input_data.data) -) -SELECT - f2.key, - f2.value -FROM - flattened f - left join lateral flatten(metric_value) f2 -where - metric_key = 'spend_zero_mean_unit_variance'; +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +-- For more details visit: +-- https://medium.com/@pkantyka/observability-of-python-code-and-application-logic-with-hamilton-ui-on-snowflake-container-services-a26693b46635 + +CREATE OR REPLACE IMAGE REPOSITORY images; +-- then docker build -t /snowflake-hamilton-ui . +-- docker login -u +-- docker push /snowflake-hamilton-ui +SHOW IMAGES IN IMAGE REPOSITORY ; + +CREATE STAGE hamilton_base ENCRYPTION = (type = 'SNOWFLAKE_SSE'); + + +CREATE SERVICE public.hamilton_ui +IN COMPUTE POOL TEST_POOL +FROM SPECIFICATION $$ +spec: + containers: + - name: hamiltonui + image: ////snowflake-hamilton-ui + volumeMounts: + - name: hamilton-basedir + mountPath: /hamilton-basedir + endpoints: + - name: entrypoint + port: 8001 + - name: hamilton + port: 8241 + public: true + volumes: + - name: hamilton-basedir + source: "@..hamilton_base" + $$ +QUERY_WAREHOUSE = +; + +CALL SYSTEM$GET_SERVICE_STATUS('..hamilton_ui'); + +CALL SYSTEM$GET_SERVICE_LOGS('..hamilton_ui', '0', 'hammiltonui', 1000); + +SHOW ENDPOINTS IN SERVICE public.hamilton_ui; + +CREATE OR REPLACE FUNCTION public.hamilton_pipeline (prj_id number, signups variant, spend variant, output_columns variant) + RETURNS VARIANT + SERVICE=public.hamilton_ui + ENDPOINT=entrypoint + AS '/echo'; + + +SELECT + public.hamilton_pipeline ( + 1, + [1, 10, 50, 100, 200, 400], + [10, 10, 20, 40, 40, 50], + [ 'spend', 'signups', 'spend_std_dev', 'spend_zero_mean_unit_variance' ] + ) as data; + +WITH input_data AS ( + SELECT + public.hamilton_pipeline ( + 1, + [1, 10, 50, 100, 200, 400], + [10, 10, 20, 40, 40, 50], + [ 'spend', 'signups', 'spend_std_dev', 'spend_zero_mean_unit_variance' ] + ) as data +), +flattened AS ( + SELECT + key AS metric_key, + value AS metric_value + FROM + input_data + left join LATERAL FLATTEN(input_data.data) +) +SELECT + * +FROM + flattened f; + +WITH input_data AS ( + SELECT + public.hamilton_pipeline ( + 1, + [1, 10, 50, 100, 200, 400], + [10, 10, 20, 40, 40, 50], + [ 'spend', 'signups', 'spend_std_dev', 'spend_zero_mean_unit_variance' ] + ) as data +), +flattened AS ( + SELECT + key AS metric_key, + value AS metric_value + FROM + input_data + left join LATERAL FLATTEN(input_data.data) +) +SELECT + f2.key, + f2.value +FROM + flattened f + left join lateral flatten(metric_value) f2 +where + metric_key = 'spend_zero_mean_unit_variance'; diff --git a/examples/spark/README.md b/examples/spark/README.md index d6eb61bc7..fe0067b63 100644 --- a/examples/spark/README.md +++ b/examples/spark/README.md @@ -1,3 +1,22 @@ + + # Scaling Apache Hamilton on Spark ## Pyspark diff --git a/examples/spark/pandas_on_spark/README.md b/examples/spark/pandas_on_spark/README.md index 3c93c5cdb..dd11e85f9 100644 --- a/examples/spark/pandas_on_spark/README.md +++ b/examples/spark/pandas_on_spark/README.md @@ -1,3 +1,22 @@ + + # Apache Hamilton on Koalas Spark 3.2+ Here we have a hello world example showing how you can diff --git a/examples/spark/pandas_on_spark/data_loaders.py b/examples/spark/pandas_on_spark/data_loaders.py index aa087248a..b189f39ee 100644 --- a/examples/spark/pandas_on_spark/data_loaders.py +++ b/examples/spark/pandas_on_spark/data_loaders.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd import pyspark.pandas as ps diff --git a/examples/spark/pandas_on_spark/run.py b/examples/spark/pandas_on_spark/run.py index 93f546313..429685711 100644 --- a/examples/spark/pandas_on_spark/run.py +++ b/examples/spark/pandas_on_spark/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import importlib import pyspark.pandas as ps diff --git a/examples/spark/pyspark/README.md b/examples/spark/pyspark/README.md index feb016639..a9bb3cfc3 100644 --- a/examples/spark/pyspark/README.md +++ b/examples/spark/pyspark/README.md @@ -1,3 +1,22 @@ + + # Apache Hamilton and Pyspark **TL;DR** Apache Hamilton now has a full pyspark integration. This enables you to build modular pyspark applications diff --git a/examples/spark/pyspark/dataflow.py b/examples/spark/pyspark/dataflow.py index 8f40f5937..f5bfed60e 100644 --- a/examples/spark/pyspark/dataflow.py +++ b/examples/spark/pyspark/dataflow.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Dict import map_transforms diff --git a/examples/spark/pyspark/map_transforms.py b/examples/spark/pyspark/map_transforms.py index 876268414..9c2553f38 100644 --- a/examples/spark/pyspark/map_transforms.py +++ b/examples/spark/pyspark/map_transforms.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from hamilton.htypes import column diff --git a/examples/spark/pyspark/notebook.ipynb b/examples/spark/pyspark/notebook.ipynb index 2b2264a9d..ee79c008c 100644 --- a/examples/spark/pyspark/notebook.ipynb +++ b/examples/spark/pyspark/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/spark/pyspark/run.py b/examples/spark/pyspark/run.py index 227f8a5a0..326a6aa90 100644 --- a/examples/spark/pyspark/run.py +++ b/examples/spark/pyspark/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import dataflow import map_transforms diff --git a/examples/spark/pyspark_feature_catalog/column_features.py b/examples/spark/pyspark_feature_catalog/column_features.py index 3917869ba..faef3835a 100644 --- a/examples/spark/pyspark_feature_catalog/column_features.py +++ b/examples/spark/pyspark_feature_catalog/column_features.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from hamilton import htypes diff --git a/examples/spark/pyspark_feature_catalog/example_usage.ipynb b/examples/spark/pyspark_feature_catalog/example_usage.ipynb index 8c1b72a4b..4ff38f04c 100644 --- a/examples/spark/pyspark_feature_catalog/example_usage.ipynb +++ b/examples/spark/pyspark_feature_catalog/example_usage.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/spark/pyspark_feature_catalog/features.py b/examples/spark/pyspark_feature_catalog/features.py index 670360239..92e45dcd4 100644 --- a/examples/spark/pyspark_feature_catalog/features.py +++ b/examples/spark/pyspark_feature_catalog/features.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pyspark.sql as ps from column_features import darkshore_flag, durotar_flag from pyspark.sql import functions as sf diff --git a/examples/spark/pyspark_udfs/README.md b/examples/spark/pyspark_udfs/README.md index c24e1fd16..a8d52b7ed 100644 --- a/examples/spark/pyspark_udfs/README.md +++ b/examples/spark/pyspark_udfs/README.md @@ -1,3 +1,22 @@ + + # Using Apache Hamilton to help manage your pyspark UDFs Here we have a version of the Apache Hamilton hello world example showing how you can take some Apache Hamilton functions and then diff --git a/examples/spark/pyspark_udfs/pandas_udfs.py b/examples/spark/pyspark_udfs/pandas_udfs.py index 2525cdd80..408a6e6d7 100644 --- a/examples/spark/pyspark_udfs/pandas_udfs.py +++ b/examples/spark/pyspark_udfs/pandas_udfs.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Pandas UDFs. Has to only contain map operations! Aggregations, filters, etc. are not supported. diff --git a/examples/spark/pyspark_udfs/run.py b/examples/spark/pyspark_udfs/run.py index 232a69896..14e7fe7e7 100644 --- a/examples/spark/pyspark_udfs/run.py +++ b/examples/spark/pyspark_udfs/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Spark driver and Hamilton driver code.""" import pandas as pd diff --git a/examples/spark/pyspark_udfs/vanilla_udfs.py b/examples/spark/pyspark_udfs/vanilla_udfs.py index 6b7c8d343..c5f822433 100644 --- a/examples/spark/pyspark_udfs/vanilla_udfs.py +++ b/examples/spark/pyspark_udfs/vanilla_udfs.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Pyspark UDFs Has to only contain map operations! diff --git a/examples/spark/tpc-h/README.md b/examples/spark/tpc-h/README.md index 18e348f2b..eb81edea3 100644 --- a/examples/spark/tpc-h/README.md +++ b/examples/spark/tpc-h/README.md @@ -1,3 +1,22 @@ + + # TPC-H We've represented a few TPC-h queries using pyspark + hamilton. diff --git a/examples/spark/tpc-h/csv_data_loaders.py b/examples/spark/tpc-h/csv_data_loaders.py index 3aca0f042..8cc328c03 100644 --- a/examples/spark/tpc-h/csv_data_loaders.py +++ b/examples/spark/tpc-h/csv_data_loaders.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os import pyspark.sql as ps diff --git a/examples/spark/tpc-h/query_1.py b/examples/spark/tpc-h/query_1.py index 673a83fef..80a16d778 100644 --- a/examples/spark/tpc-h/query_1.py +++ b/examples/spark/tpc-h/query_1.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import datetime import pandas as pd diff --git a/examples/spark/tpc-h/query_12.py b/examples/spark/tpc-h/query_12.py index 4c5e2c42f..696ec1ffc 100644 --- a/examples/spark/tpc-h/query_12.py +++ b/examples/spark/tpc-h/query_12.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd import pyspark.sql as ps from pyspark.sql import functions as F diff --git a/examples/spark/tpc-h/query_8.py b/examples/spark/tpc-h/query_8.py index d6957d7b3..97ea6eb3b 100644 --- a/examples/spark/tpc-h/query_8.py +++ b/examples/spark/tpc-h/query_8.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd import pyspark.sql as ps import pyspark.sql.functions as F diff --git a/examples/spark/tpc-h/run.py b/examples/spark/tpc-h/run.py index 7cc7af6e0..10435b839 100644 --- a/examples/spark/tpc-h/run.py +++ b/examples/spark/tpc-h/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import click exec("from hamilton.plugins import h_spark") diff --git a/examples/spark/world_of_warcraft/README.md b/examples/spark/world_of_warcraft/README.md index ed8eab54a..84b8dc675 100644 --- a/examples/spark/world_of_warcraft/README.md +++ b/examples/spark/world_of_warcraft/README.md @@ -1,3 +1,22 @@ + + # Example Feature Pipeline for both Spark and Pandas This folder contains code to create features from World of Warcraft data. diff --git a/examples/spark/world_of_warcraft/world_of_warcraft__pandas.ipynb b/examples/spark/world_of_warcraft/world_of_warcraft__pandas.ipynb index 5e5e1ee67..8b8d5b3f8 100644 --- a/examples/spark/world_of_warcraft/world_of_warcraft__pandas.ipynb +++ b/examples/spark/world_of_warcraft/world_of_warcraft__pandas.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/spark/world_of_warcraft/world_of_warcraft__spark_v1.ipynb b/examples/spark/world_of_warcraft/world_of_warcraft__spark_v1.ipynb index 5ef0821cc..e39984f83 100644 --- a/examples/spark/world_of_warcraft/world_of_warcraft__spark_v1.ipynb +++ b/examples/spark/world_of_warcraft/world_of_warcraft__spark_v1.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/spark/world_of_warcraft/world_of_warcraft__spark_v2.ipynb b/examples/spark/world_of_warcraft/world_of_warcraft__spark_v2.ipynb index abc426320..86e064921 100644 --- a/examples/spark/world_of_warcraft/world_of_warcraft__spark_v2.ipynb +++ b/examples/spark/world_of_warcraft/world_of_warcraft__spark_v2.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/spark/world_of_warcraft/zone_features__common.py b/examples/spark/world_of_warcraft/zone_features__common.py index c09ddacc1..308c98aa1 100644 --- a/examples/spark/world_of_warcraft/zone_features__common.py +++ b/examples/spark/world_of_warcraft/zone_features__common.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from hamilton import htypes diff --git a/examples/spark/world_of_warcraft/zone_features__pandas.py b/examples/spark/world_of_warcraft/zone_features__pandas.py index c1d47eb54..8c0587137 100644 --- a/examples/spark/world_of_warcraft/zone_features__pandas.py +++ b/examples/spark/world_of_warcraft/zone_features__pandas.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from hamilton.function_modifiers import extract_columns diff --git a/examples/spark/world_of_warcraft/zone_features__spark_v1.py b/examples/spark/world_of_warcraft/zone_features__spark_v1.py index 93ea23886..305e80f8d 100644 --- a/examples/spark/world_of_warcraft/zone_features__spark_v1.py +++ b/examples/spark/world_of_warcraft/zone_features__spark_v1.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pyspark.sql as ps from pyspark.sql import functions as sf diff --git a/examples/spark/world_of_warcraft/zone_features__spark_v2.py b/examples/spark/world_of_warcraft/zone_features__spark_v2.py index 0dd014ac2..d1b58dfa5 100644 --- a/examples/spark/world_of_warcraft/zone_features__spark_v2.py +++ b/examples/spark/world_of_warcraft/zone_features__spark_v2.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pyspark.sql as ps from pyspark.sql import functions as sf from zone_features__common import ( diff --git a/examples/streamlit/README.md b/examples/streamlit/README.md index bda7d070d..2120a323e 100644 --- a/examples/streamlit/README.md +++ b/examples/streamlit/README.md @@ -1,3 +1,22 @@ + + # Streamlit + Apache Hamilton > This example accompanies the documentation page for [Streamlit](https://hamilton.apache.org/integrations/streamlit/) integration. diff --git a/examples/streamlit/app.py b/examples/streamlit/app.py index bc9bef89a..e816722ef 100644 --- a/examples/streamlit/app.py +++ b/examples/streamlit/app.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Optional import logic diff --git a/examples/streamlit/logic.py b/examples/streamlit/logic.py index 92a1a9738..ea69e3ba6 100644 --- a/examples/streamlit/logic.py +++ b/examples/streamlit/logic.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd import plotly.express as px from plotly.graph_objs import Figure diff --git a/examples/styling_visualization/README.md b/examples/styling_visualization/README.md index a3b2ebf56..e018ec8b5 100644 --- a/examples/styling_visualization/README.md +++ b/examples/styling_visualization/README.md @@ -1,3 +1,22 @@ + + # Style your dataflow visualizations A core feature of Apache Hamilton is the ability to generate visualizations directly from your code. By default, each nodes is labelled with its name and type and is stylized (shape, color, outline, etc.). diff --git a/examples/styling_visualization/dag.png b/examples/styling_visualization/dag.png index ceda0c39e..3e710da29 100644 Binary files a/examples/styling_visualization/dag.png and b/examples/styling_visualization/dag.png differ diff --git a/examples/styling_visualization/data_loading.py b/examples/styling_visualization/data_loading.py index 49390efd8..b053afee4 100644 --- a/examples/styling_visualization/data_loading.py +++ b/examples/styling_visualization/data_loading.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import List import pandas as pd diff --git a/examples/styling_visualization/features.py b/examples/styling_visualization/features.py index 1152da7b1..c185ea73a 100644 --- a/examples/styling_visualization/features.py +++ b/examples/styling_visualization/features.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import pandas as pd from sklearn import preprocessing diff --git a/examples/styling_visualization/model_pipeline.py b/examples/styling_visualization/model_pipeline.py index 5fe71a3c6..e25c666c4 100644 --- a/examples/styling_visualization/model_pipeline.py +++ b/examples/styling_visualization/model_pipeline.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Dict, Union import pandas as pd diff --git a/examples/styling_visualization/run.py b/examples/styling_visualization/run.py index e336f4635..13966eef4 100644 --- a/examples/styling_visualization/run.py +++ b/examples/styling_visualization/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import data_loading import features import model_pipeline diff --git a/examples/styling_visualization/sets.py b/examples/styling_visualization/sets.py index 2482b0330..480f5003a 100644 --- a/examples/styling_visualization/sets.py +++ b/examples/styling_visualization/sets.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from hamilton.function_modifiers import tag diff --git a/examples/vaex/README.md b/examples/vaex/README.md index 9af30a4ba..86d87e41f 100644 --- a/examples/vaex/README.md +++ b/examples/vaex/README.md @@ -1,3 +1,22 @@ + + # Vaex Example This is a simple example taking the Apache Hamilton hello world and converting it to run with Vaex. diff --git a/examples/vaex/my_functions.py b/examples/vaex/my_functions.py index f45dfcb37..0d2615f75 100644 --- a/examples/vaex/my_functions.py +++ b/examples/vaex/my_functions.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import numpy as np import pandas as pd import vaex diff --git a/examples/vaex/my_script.py b/examples/vaex/my_script.py index eb5464ca8..2190ff14e 100644 --- a/examples/vaex/my_script.py +++ b/examples/vaex/my_script.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging import sys diff --git a/examples/vaex/notebook.ipynb b/examples/vaex/notebook.ipynb index a375d2b65..624a9f870 100644 --- a/examples/vaex/notebook.ipynb +++ b/examples/vaex/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/validate_examples.py b/examples/validate_examples.py index 5ccf69868..133f0bcec 100644 --- a/examples/validate_examples.py +++ b/examples/validate_examples.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import annotations import argparse @@ -35,16 +52,23 @@ def validate_notebook(notebook_path: pathlib.Path) -> int: in Google Colab, and that the second cell has badges to open the notebook in Google Colab and view the source on GitHub. + This function handles notebooks with or without Apache license headers: + - If first cell is Apache license (markdown): check cells 2 and 3 + - Otherwise: check cells 1 and 2 (original behavior) + NOTE. For faster notebook startup (especially on Colab), we should disable plugin autoloading .. code-block:: python - #%% CELL_1 + #%% CELL_1 (optional) + # Apache License header (markdown) + + #%% CELL_2 (or CELL_1 if no license) # Execute this cell to install dependencies %pip install sf-hamilton[visualization] matplotlib - #%% CELL_2 + #%% CELL_3 (or CELL_2 if no license) # Title of the notebook ![Colab badge](colab_url) ![GitHub badge](github_url) """ @@ -57,7 +81,6 @@ def validate_notebook(notebook_path: pathlib.Path) -> int: return FAILURE first_cell = notebook.cells[0] - second_cell = notebook.cells[1] issues = [] @@ -66,29 +89,54 @@ def validate_notebook(notebook_path: pathlib.Path) -> int: logger.info(f"Ignoring because path is excluded: `{notebook_path}`") return SUCCESS - if first_cell.cell_type != "code": - issues.append("The first cell should be a code cell to set up the notebook.") - RETURN_VALUE |= FAILURE + # Check if the first cell is an Apache license header + has_license_header = ( + first_cell.cell_type == "markdown" and "Apache License" in first_cell.source + ) - if "%pip install" not in first_cell.source: - issues.append( - "In the first cell, use the `%pip` magic to install dependencies for the notebook." - ) + # Determine which cells to check based on whether there's a license header + if has_license_header: + # License header present: check cells 2 and 3 (indices 1 and 2) + setup_cell = notebook.cells[1] if len(notebook.cells) > 1 else None + title_cell = notebook.cells[2] if len(notebook.cells) > 2 else None + else: + # No license header: check cells 1 and 2 (indices 0 and 1) - original behavior + setup_cell = first_cell + title_cell = notebook.cells[1] if len(notebook.cells) > 1 else None + + # Validate setup cell + if setup_cell is None: + issues.append("Notebook must have at least a setup cell.") RETURN_VALUE |= FAILURE - - if second_cell.cell_type != "markdown": - issues.append( - "The second cell should be markdown with the title, badges, and introduction." - ) + else: + if setup_cell.cell_type != "code": + issues.append("The setup cell should be a code cell.") + RETURN_VALUE |= FAILURE + + if "%pip install" not in setup_cell.source: + issues.append( + "In the setup cell, use the `%pip` magic to install dependencies for the notebook." + ) + RETURN_VALUE |= FAILURE + + # Validate title cell + if title_cell is None: + issues.append("Notebook must have a title cell.") RETURN_VALUE |= FAILURE + else: + if title_cell.cell_type != "markdown": + issues.append( + "The title cell should be markdown with the title, badges, and introduction." + ) + RETURN_VALUE |= FAILURE - if _create_colab_badge(notebook_path) not in second_cell.source: - issues.append("Missing badge to open notebook in Google Colab.") - RETURN_VALUE |= FAILURE + if _create_colab_badge(notebook_path) not in title_cell.source: + issues.append("Missing badge to open notebook in Google Colab.") + RETURN_VALUE |= FAILURE - if _create_github_badge(notebook_path) not in second_cell.source: - issues.append("Missing badge to view source on GitHub.") - RETURN_VALUE |= FAILURE + if _create_github_badge(notebook_path) not in title_cell.source: + issues.append("Missing badge to view source on GitHub.") + RETURN_VALUE |= FAILURE if RETURN_VALUE == FAILURE: joined_issues = "\n\t".join(issues) @@ -98,18 +146,32 @@ def validate_notebook(notebook_path: pathlib.Path) -> int: def insert_setup_cell(path: pathlib.Path): - """Insert a setup cell at the top of a notebook. + """Insert a setup cell at the top of a notebook (or after license header if present). Calling this multiple times will add multiple setup cells. - This should be called before adding badges to the second cell, + This should be called before adding badges to the title cell, which is expected to be markdown. + + If the first cell is an Apache license header (markdown), the setup cell + is inserted at position 1 (after the license). Otherwise, it's inserted at position 0. """ notebook = nbformat.read(path, as_version=4) setup_cell = nbformat.v4.new_code_cell( "# Execute this cell to install dependencies\n%pip install sf-hamilton[visualization]" ) - notebook.cells.insert(0, setup_cell) + + # Check if first cell is a license header + first_cell = notebook.cells[0] if len(notebook.cells) > 0 else None + has_license_header = ( + first_cell is not None + and first_cell.cell_type == "markdown" + and "Apache License" in first_cell.source + ) + + # Insert after license header if present, otherwise at the beginning + insert_position = 1 if has_license_header else 0 + notebook.cells.insert(insert_position, setup_cell) # cleanup required to avoid nbformat warnings for cell in notebook.cells: @@ -120,24 +182,42 @@ def insert_setup_cell(path: pathlib.Path): def add_badges_to_title(path: pathlib.Path): - """Add badges to the second cell of the notebook. + """Add badges to the title cell of the notebook. + + This should be called after inserting the setup cell. - This should be called after inserting the setup cell, - which should be the first cell of the notebook. + The title cell is expected to be: + - Cell 2 (index 2) if there's a license header at cell 0 + - Cell 1 (index 1) if there's no license header """ notebook = nbformat.read(path, as_version=4) - if notebook.cells[1].cell_type != "markdown": + + # Check if first cell is a license header + first_cell = notebook.cells[0] if len(notebook.cells) > 0 else None + has_license_header = ( + first_cell is not None + and first_cell.cell_type == "markdown" + and "Apache License" in first_cell.source + ) + + # Determine which cell is the title cell + title_cell_index = 2 if has_license_header else 1 + + if len(notebook.cells) <= title_cell_index: + return + + if notebook.cells[title_cell_index].cell_type != "markdown": return updated_content = "" - for idx, line in enumerate(notebook.cells[1].source.splitlines()): + for idx, line in enumerate(notebook.cells[title_cell_index].source.splitlines()): if idx == 0: updated_content += f"{line} {_create_colab_badge(path)} {_create_github_badge(path)}\n" else: updated_content += f"\n{line}" - notebook.cells[1].update(source=updated_content) + notebook.cells[title_cell_index].update(source=updated_content) nbformat.write(notebook, path) diff --git a/examples/validation/static_validator/README.md b/examples/validation/static_validator/README.md index 7af314e35..6f3b12d39 100644 --- a/examples/validation/static_validator/README.md +++ b/examples/validation/static_validator/README.md @@ -1,3 +1,22 @@ + + # Example showing graph validation This uses the StaticValidator class to validate a graph. diff --git a/examples/validation/static_validator/bad_module.py b/examples/validation/static_validator/bad_module.py index 776c9898a..6a0334c94 100644 --- a/examples/validation/static_validator/bad_module.py +++ b/examples/validation/static_validator/bad_module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from hamilton.function_modifiers import tag diff --git a/examples/validation/static_validator/good_module.py b/examples/validation/static_validator/good_module.py index 8621c354b..fe7664752 100644 --- a/examples/validation/static_validator/good_module.py +++ b/examples/validation/static_validator/good_module.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from hamilton.function_modifiers import tag diff --git a/examples/validation/static_validator/notebook.ipynb b/examples/validation/static_validator/notebook.ipynb index 9749db75f..c3da8162e 100644 --- a/examples/validation/static_validator/notebook.ipynb +++ b/examples/validation/static_validator/notebook.ipynb @@ -1,5 +1,12 @@ { "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Licensed to the Apache Software Foundation (ASF) under one\nor more contributor license agreements. See the NOTICE file\ndistributed with this work for additional information\nregarding copyright ownership. The ASF licenses this file\nto you under the Apache License, Version 2.0 (the\n\"License\"); you may not use this file except in compliance\nwith the License. You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\nUnless required by applicable law or agreed to in writing,\nsoftware distributed under the License is distributed on an\n\"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\nKIND, either express or implied. See the License for the\nspecific language governing permissions and limitations\nunder the License." + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/examples/validation/static_validator/run.py b/examples/validation/static_validator/run.py index 8d9183ce8..fbee74a65 100644 --- a/examples/validation/static_validator/run.py +++ b/examples/validation/static_validator/run.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from typing import Optional from hamilton.graph_types import HamiltonNode diff --git a/hamilton/experimental/databackend.py b/hamilton/experimental/databackend.py index 2eae6fbe7..ff1d1ebea 100644 --- a/hamilton/experimental/databackend.py +++ b/hamilton/experimental/databackend.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Below is a file from `databackend` v0.0.2 found on GitHub permalink: https://github.com/machow/databackend/blob/214832bab6990098d52a4a2fae935f5d9ae8dc3c/databackend/__init__.py """ diff --git a/scripts/add_license_headers.py b/scripts/add_license_headers.py new file mode 100755 index 000000000..259992c1b --- /dev/null +++ b/scripts/add_license_headers.py @@ -0,0 +1,323 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Script to add Apache 2 license headers to files in the Hamilton repository.""" + +import json +import sys +from pathlib import Path +from typing import List + +# TODO: Simplify this script if we add more file types. Since the license text +# (including line breaks) stays the same, we really just need different comment +# character insertion helpers based on file extension rather than duplicating +# the full license text for each file type. + +# Apache 2 license header for Python files +PYTHON_LICENSE_HEADER = """# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" + +# Apache 2 license header for Markdown files (using HTML comments) +MARKDOWN_LICENSE_HEADER = """ + +""" + +# Apache 2 license header text for Jupyter notebooks (as markdown cell content) +NOTEBOOK_LICENSE_TEXT = """Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License.""" + +# Apache 2 license header for SQL files (using -- comments) +SQL_LICENSE_HEADER = """-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +""" + + +def add_license_to_python(content: str) -> str: + """Add Apache 2 license header to Python file content.""" + # Handle shebang lines - preserve them at the top + lines = content.split("\n", 1) + if lines[0].startswith("#!"): + # File has shebang, add license after it + if len(lines) > 1: + return lines[0] + "\n" + PYTHON_LICENSE_HEADER + lines[1] + else: + return lines[0] + "\n" + PYTHON_LICENSE_HEADER + else: + # No shebang, add license at the beginning + return PYTHON_LICENSE_HEADER + content + + +def add_license_to_markdown(content: str) -> str: + """Add Apache 2 license header to Markdown file content.""" + return MARKDOWN_LICENSE_HEADER + content + + +def add_license_to_notebook(content: str) -> str: + """Add Apache 2 license header to Jupyter notebook.""" + try: + notebook = json.loads(content) + except json.JSONDecodeError as e: + raise ValueError(f"Invalid notebook JSON: {e}") from e + + # Create a new markdown cell with the license + license_cell = {"cell_type": "markdown", "metadata": {}, "source": [NOTEBOOK_LICENSE_TEXT]} + + # Insert at the beginning + if "cells" not in notebook: + notebook["cells"] = [] + + notebook["cells"].insert(0, license_cell) + + return json.dumps(notebook, indent=1, ensure_ascii=False) + + +def add_license_to_shell(content: str) -> str: + """Add Apache 2 license header to shell script or Dockerfile. + + Uses same logic as Python files (# comments, handle shebang). + """ + # Handle shebang lines - preserve them at the top + lines = content.split("\n", 1) + if lines[0].startswith("#!"): + # File has shebang, add license after it + if len(lines) > 1: + return lines[0] + "\n" + PYTHON_LICENSE_HEADER + lines[1] + else: + return lines[0] + "\n" + PYTHON_LICENSE_HEADER + else: + # No shebang, add license at the beginning + return PYTHON_LICENSE_HEADER + content + + +def add_license_to_sql(content: str) -> str: + """Add Apache 2 license header to SQL file content.""" + return SQL_LICENSE_HEADER + content + + +def add_license_header(file_path: Path, dry_run: bool = False) -> bool: + """Add Apache 2 license header to a file. + + Args: + file_path: Path to the file + dry_run: If True, only print what would be done without modifying files + + Returns: + True if header was added (or would be added in dry run), False otherwise + """ + try: + with open(file_path, "r", encoding="utf-8") as f: + content = f.read() + except (UnicodeDecodeError, PermissionError) as e: + print(f" ✗ Error reading {file_path}: {e}") + return False + + # Check if file already has a license header (check first 20 lines only) + first_lines = "\n".join(content.split("\n")[:20]) + if ( + "Licensed to the Apache Software Foundation" in first_lines + or "Apache License" in first_lines + ): + print(f" ↷ Skipping {file_path} (already has license header)") + return False + + # Determine file type and add appropriate header + try: + if file_path.suffix == ".py": + new_content = add_license_to_python(content) + elif file_path.suffix == ".md": + new_content = add_license_to_markdown(content) + elif file_path.suffix == ".ipynb": + new_content = add_license_to_notebook(content) + elif file_path.suffix == ".sh": + new_content = add_license_to_shell(content) + elif file_path.suffix == ".sql": + new_content = add_license_to_sql(content) + elif file_path.name == "Dockerfile": + # Dockerfiles use # comments like shell scripts + new_content = add_license_to_shell(content) + elif file_path.name == "README": + # README files without extension are usually markdown + new_content = add_license_to_markdown(content) + else: + print(f" ✗ Unsupported file type: {file_path.suffix} ({file_path.name})") + return False + except Exception as e: + print(f" ✗ Error processing {file_path}: {e}") + return False + + if dry_run: + print(f" ✓ Would add license header to {file_path}") + return True + + # Write the modified content back + try: + with open(file_path, "w", encoding="utf-8") as f: + f.write(new_content) + print(f" ✓ Added license header to {file_path}") + return True + except (PermissionError, OSError) as e: + print(f" ✗ Error writing {file_path}: {e}") + return False + + +def main(): + """Main function.""" + # Get repository root (parent of scripts directory) + repo_root = Path(__file__).parent.parent + + # Check for dry-run flag + dry_run = "--dry-run" in sys.argv or "-n" in sys.argv + + if dry_run: + print("DRY RUN MODE - No files will be modified\n") + + # Read the list of files from command line or use default + if len(sys.argv) > 1 and not sys.argv[-1].startswith("-"): + list_file = repo_root / sys.argv[-1] + else: + # Try combined list first, fall back to missing_license_headers.txt + list_file = repo_root / "missing_license_headers_combined.txt" + if not list_file.exists(): + list_file = repo_root / "missing_license_headers.txt" + + if not list_file.exists(): + print(f"Error: {list_file} not found!") + print("Run check_license_headers.py first to generate the list.") + print("Or specify a file list: python add_license_headers.py ") + return 1 + + print(f"Using file list: {list_file.name}\n") + + # Parse the file list + files_to_update: List[Path] = [] + with open(list_file, "r") as f: + for line in f: + original_line = line + line = line.strip() + # Skip header lines and empty lines + if ( + not line + or line.startswith("Checking") + or line.startswith("Extensions") + or line.startswith("Found") + ): + continue + # Skip lines that look like section headers (not starting with space and not a file path) + if ( + not original_line.startswith(" ") + and not line.startswith("examples") + and not line.startswith("hamilton") + and "/" not in line + ): + continue + # Remove leading spaces + file_path = line + full_path = repo_root / file_path + if full_path.exists(): + files_to_update.append(full_path) + + if not files_to_update: + print("No files to update!") + return 0 + + print(f"Found {len(files_to_update)} files to add license headers to\n") + + # Process each file + success_count = 0 + skip_count = 0 + + for file_path in files_to_update: + result = add_license_header(file_path, dry_run=dry_run) + if result: + success_count += 1 + else: + # Could be a skip or an error - check the output + skip_count += 1 + + # Print summary + print(f"\n{'Would add' if dry_run else 'Added'} license headers to {success_count} files") + if skip_count > 0: + print(f"Skipped {skip_count} files") + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check_license_headers.py b/scripts/check_license_headers.py new file mode 100755 index 000000000..fc67379e8 --- /dev/null +++ b/scripts/check_license_headers.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Script to find files missing Apache 2 license headers in the Hamilton repository.""" + +import sys +from pathlib import Path +from typing import List, Set + +# License header patterns to check for +LICENSE_PATTERNS = [ + "Licensed to the Apache Software Foundation (ASF)", + "Apache License, Version 2.0", +] + +# File extensions to check +PYTHON_EXTENSIONS = {".py"} +MARKDOWN_EXTENSIONS = {".md"} +NOTEBOOK_EXTENSIONS = {".ipynb"} +SHELL_EXTENSIONS = {".sh"} +SQL_EXTENSIONS = {".sql"} +TYPESCRIPT_EXTENSIONS = {".ts", ".tsx"} +JAVASCRIPT_EXTENSIONS = {".js", ".jsx"} +ALL_EXTENSIONS = ( + PYTHON_EXTENSIONS + | MARKDOWN_EXTENSIONS + | NOTEBOOK_EXTENSIONS + | SHELL_EXTENSIONS + | SQL_EXTENSIONS + | TYPESCRIPT_EXTENSIONS + | JAVASCRIPT_EXTENSIONS +) + +# Directories to skip +SKIP_DIRS = { + ".git", + "__pycache__", + "node_modules", + ".pytest_cache", + ".mypy_cache", + ".tox", + "venv", + ".venv", + "build", + "dist", + "*.egg-info", + ".eggs", + "htmlcov", + ".coverage", + ".claude", +} + + +def should_skip_path(path: Path) -> bool: + """Check if a path should be skipped.""" + # Skip if any parent directory is in SKIP_DIRS + for part in path.parts: + if part in SKIP_DIRS or part.startswith("."): + return True + + # Skip documentation snippet files (they're embedded in docs via literalinclude) + path_str = str(path) + if "docs" in path.parts and "_snippets" in path_str: + return True + if "docs/code-comparisons" in path_str and "snippets" in path_str: + return True + + return False + + +def has_license_header(file_path: Path, num_lines: int = 20) -> bool: + """Check if a file has the Apache 2 license header.""" + try: + with open(file_path, "r", encoding="utf-8") as f: + content = "".join(f.readlines()[:num_lines]) + + # Check if all license patterns are present in the first N lines + return all(pattern in content for pattern in LICENSE_PATTERNS) + except (UnicodeDecodeError, PermissionError): + # Skip files that can't be read as text + return True # Assume they're fine to avoid false positives + + +def find_files_without_license( + root_dir: Path, extensions: Set[str] = ALL_EXTENSIONS, include_special: bool = True +) -> List[Path]: + """Find all files without Apache 2 license headers. + + Args: + root_dir: Root directory to search + extensions: Set of file extensions to check + include_special: Whether to include special files like Dockerfile and README (no extension) + """ + files_without_license = [] + + # Special files to check (by exact name, not extension) + SPECIAL_FILES = {"Dockerfile", "README"} + + for file_path in root_dir.rglob("*"): + # Skip directories + if file_path.is_dir(): + continue + + # Check if file matches by extension or by special name + matches_extension = file_path.suffix in extensions + matches_special = include_special and file_path.name in SPECIAL_FILES + + if not (matches_extension or matches_special): + continue + + # Skip if in excluded paths + if should_skip_path(file_path): + continue + + # Check for license header + if not has_license_header(file_path): + files_without_license.append(file_path) + + return sorted(files_without_license) + + +def main(): + """Main function.""" + # Get repository root (parent of scripts directory) + repo_root = Path(__file__).parent.parent + + # Allow specifying extensions via command line + if len(sys.argv) > 1: + extension_arg = sys.argv[1] + if extension_arg == "py": + extensions = PYTHON_EXTENSIONS + elif extension_arg == "md": + extensions = MARKDOWN_EXTENSIONS + elif extension_arg == "ipynb": + extensions = NOTEBOOK_EXTENSIONS + elif extension_arg == "sh": + extensions = SHELL_EXTENSIONS + elif extension_arg == "sql": + extensions = SQL_EXTENSIONS + elif extension_arg == "ts": + extensions = TYPESCRIPT_EXTENSIONS + elif extension_arg == "js": + extensions = JAVASCRIPT_EXTENSIONS + elif extension_arg == "special": + # Check only Dockerfile and README files + extensions = set() + else: + extensions = ALL_EXTENSIONS + else: + extensions = ALL_EXTENSIONS + + # Only include special files (Dockerfile, README) if checking all or "special" type + include_special = len(sys.argv) <= 1 or ( + len(sys.argv) > 1 and sys.argv[1] in ["special", "all"] + ) + + print(f"Checking for Apache 2 license headers in {repo_root}") + if extensions: + print(f"Extensions: {', '.join(sorted(extensions))}") + if include_special: + print("Including: Dockerfile, README files") + print() + + files_without_license = find_files_without_license(repo_root, extensions, include_special) + + if not files_without_license: + print("✓ All files have license headers!") + return 0 + + print(f"Found {len(files_without_license)} files without Apache 2 license headers:\n") + + for file_path in files_without_license: + # Print relative path from repo root + try: + rel_path = file_path.relative_to(repo_root) + print(f" {rel_path}") + except ValueError: + print(f" {file_path}") + + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_memory.py b/scripts/test_memory.py index 77de31bfd..44d813b23 100644 --- a/scripts/test_memory.py +++ b/scripts/test_memory.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Script to test memory. Run with mprof: diff --git a/scripts/update_blogs_in_learning_resources.py b/scripts/update_blogs_in_learning_resources.py index 4fb7a1185..dd6601ca0 100644 --- a/scripts/update_blogs_in_learning_resources.py +++ b/scripts/update_blogs_in_learning_resources.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ 📜 DAGWorks Blog Archive Updater Script diff --git a/ui/backend/server/trackingserver_base/tests.py b/ui/backend/server/trackingserver_base/tests.py index a79ca8be5..6d9186750 100644 --- a/ui/backend/server/trackingserver_base/tests.py +++ b/ui/backend/server/trackingserver_base/tests.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # from django.test import TestCase # Create your tests here. diff --git a/ui/sdk/tests/conftest.py b/ui/sdk/tests/conftest.py index 04d11ddc9..ce5106797 100644 --- a/ui/sdk/tests/conftest.py +++ b/ui/sdk/tests/conftest.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from hamilton import telemetry telemetry.disable_telemetry() diff --git a/ui/sdk/tests/resources/basic_dag_with_config.py b/ui/sdk/tests/resources/basic_dag_with_config.py index 5c10b41a1..098fd34e4 100644 --- a/ui/sdk/tests/resources/basic_dag_with_config.py +++ b/ui/sdk/tests/resources/basic_dag_with_config.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from hamilton.function_modifiers import config diff --git a/ui/sdk/tests/resources/parallel_dag.py b/ui/sdk/tests/resources/parallel_dag.py index d6ff78e73..f17fa527a 100644 --- a/ui/sdk/tests/resources/parallel_dag.py +++ b/ui/sdk/tests/resources/parallel_dag.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import dataclasses import os from typing import List diff --git a/ui/sdk/tests/resources/parallel_dag_error.py b/ui/sdk/tests/resources/parallel_dag_error.py index 9b1715402..bea7f4441 100644 --- a/ui/sdk/tests/resources/parallel_dag_error.py +++ b/ui/sdk/tests/resources/parallel_dag_error.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import dataclasses import os from typing import List diff --git a/ui/sdk/tests/test_adapters.py b/ui/sdk/tests/test_adapters.py index cac37949d..65d439fb2 100644 --- a/ui/sdk/tests/test_adapters.py +++ b/ui/sdk/tests/test_adapters.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import os.path import pytest diff --git a/ui/sdk/tests/test_driver.py b/ui/sdk/tests/test_driver.py index 7a2950998..f32180aaf 100644 --- a/ui/sdk/tests/test_driver.py +++ b/ui/sdk/tests/test_driver.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import hashlib import logging from types import ModuleType diff --git a/ui/sdk/tests/test_package_to_hash/submodule1.py b/ui/sdk/tests/test_package_to_hash/submodule1.py index 5ddf34917..11bdf850b 100644 --- a/ui/sdk/tests/test_package_to_hash/submodule1.py +++ b/ui/sdk/tests/test_package_to_hash/submodule1.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import hashlib from . import subpackage diff --git a/ui/sdk/tests/test_package_to_hash/submodule2.py b/ui/sdk/tests/test_package_to_hash/submodule2.py index ecef99526..5f7127240 100644 --- a/ui/sdk/tests/test_package_to_hash/submodule2.py +++ b/ui/sdk/tests/test_package_to_hash/submodule2.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pprint from . import subpackage diff --git a/ui/sdk/tests/test_tracking.py b/ui/sdk/tests/test_tracking.py index 983f7875f..7fb2cc023 100644 --- a/ui/sdk/tests/test_tracking.py +++ b/ui/sdk/tests/test_tracking.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import functools from datetime import datetime, timezone from typing import Any, Callable, Dict, List diff --git a/ui/sdk/tests/tracking/test_constants.py b/ui/sdk/tests/tracking/test_constants.py index 9ccab2ece..3ba5db5ac 100644 --- a/ui/sdk/tests/tracking/test_constants.py +++ b/ui/sdk/tests/tracking/test_constants.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import configparser from hamilton_sdk.tracking import constants diff --git a/ui/sdk/tests/tracking/test_dataframe_stats.py b/ui/sdk/tests/tracking/test_dataframe_stats.py index 8e53f214d..8a996fa07 100644 --- a/ui/sdk/tests/tracking/test_dataframe_stats.py +++ b/ui/sdk/tests/tracking/test_dataframe_stats.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import math import numpy as np diff --git a/ui/sdk/tests/tracking/test_ibis_stats.py b/ui/sdk/tests/tracking/test_ibis_stats.py index b1f9808a2..dd4dc6ea4 100644 --- a/ui/sdk/tests/tracking/test_ibis_stats.py +++ b/ui/sdk/tests/tracking/test_ibis_stats.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import ibis import pandas as pd from hamilton_sdk.tracking import ibis_stats diff --git a/ui/sdk/tests/tracking/test_langchain_stats.py b/ui/sdk/tests/tracking/test_langchain_stats.py index 34eee23ac..dfcd3f08b 100644 --- a/ui/sdk/tests/tracking/test_langchain_stats.py +++ b/ui/sdk/tests/tracking/test_langchain_stats.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from hamilton_sdk.tracking import langchain_stats from langchain_core import documents as lc_documents from langchain_core import messages as lc_messages diff --git a/ui/sdk/tests/tracking/test_pandas_col_stats.py b/ui/sdk/tests/tracking/test_pandas_col_stats.py index 550c736ee..684db46a5 100644 --- a/ui/sdk/tests/tracking/test_pandas_col_stats.py +++ b/ui/sdk/tests/tracking/test_pandas_col_stats.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Module for testing pandas column stats.""" import pandas as pd diff --git a/ui/sdk/tests/tracking/test_pandas_stats.py b/ui/sdk/tests/tracking/test_pandas_stats.py index da4903015..2eb2721b5 100644 --- a/ui/sdk/tests/tracking/test_pandas_stats.py +++ b/ui/sdk/tests/tracking/test_pandas_stats.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd from hamilton_sdk.tracking import pandas_stats as ps diff --git a/ui/sdk/tests/tracking/test_polars_col_stats.py b/ui/sdk/tests/tracking/test_polars_col_stats.py index 3f3a976a4..f80c12b86 100644 --- a/ui/sdk/tests/tracking/test_polars_col_stats.py +++ b/ui/sdk/tests/tracking/test_polars_col_stats.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Module for testing pandas column stats.""" import polars as pl diff --git a/ui/sdk/tests/tracking/test_polars_stats.py b/ui/sdk/tests/tracking/test_polars_stats.py index a8a9d43a9..141e4d90d 100644 --- a/ui/sdk/tests/tracking/test_polars_stats.py +++ b/ui/sdk/tests/tracking/test_polars_stats.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from datetime import date import polars as pl from hamilton_sdk.tracking import polars_stats as ps diff --git a/ui/sdk/tests/tracking/test_pydantic_stats.py b/ui/sdk/tests/tracking/test_pydantic_stats.py index f6703a39d..5059a840c 100644 --- a/ui/sdk/tests/tracking/test_pydantic_stats.py +++ b/ui/sdk/tests/tracking/test_pydantic_stats.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from hamilton_sdk.tracking import pydantic_stats from pydantic import BaseModel diff --git a/ui/sdk/tests/tracking/test_pyspark_stats.py b/ui/sdk/tests/tracking/test_pyspark_stats.py index 694afc34f..2e22c66d0 100644 --- a/ui/sdk/tests/tracking/test_pyspark_stats.py +++ b/ui/sdk/tests/tracking/test_pyspark_stats.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pandas as pd import pyspark.sql as ps from hamilton_sdk.tracking import pyspark_stats diff --git a/ui/sdk/tests/tracking/test_runs.py b/ui/sdk/tests/tracking/test_runs.py index 7feee0b44..ee615b78b 100644 --- a/ui/sdk/tests/tracking/test_runs.py +++ b/ui/sdk/tests/tracking/test_runs.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import json from datetime import date from typing import Any diff --git a/ui/sdk/tests/tracking/test_stats.py b/ui/sdk/tests/tracking/test_stats.py index 7d030090a..fbe634d0d 100644 --- a/ui/sdk/tests/tracking/test_stats.py +++ b/ui/sdk/tests/tracking/test_stats.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import sys from collections import namedtuple from typing import NamedTuple diff --git a/ui/sdk/tests/tracking/test_utils.py b/ui/sdk/tests/tracking/test_utils.py index 62cafb87f..9a8951ab7 100644 --- a/ui/sdk/tests/tracking/test_utils.py +++ b/ui/sdk/tests/tracking/test_utils.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import dataclasses import datetime from enum import Enum diff --git a/writeups/garbage_collection/memory_test.py b/writeups/garbage_collection/memory_test.py index 92b4f97a9..e42702180 100644 --- a/writeups/garbage_collection/memory_test.py +++ b/writeups/garbage_collection/memory_test.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """ Script to test memory. Run with mprof: