Added reproducible wheel and egg sources (#1832)

This PR adds reproducible wheel and egg builder.

---------

Co-authored-by: Cor <[email protected]>

Author: nfx

tests/integration/source_code/samples/library-egg/demo_egg-0.0.1-py3.6.egg

@@ -229,8 +229,8 @@ def test_workflow_linter_lints_job_with_egg_dependency(
     make_notebook,
     make_directory,
 ):
-    expected_problem_message = "Could not locate import: pkgdir"
-    egg_file = Path(__file__).parent / "samples" / "library-egg" / "demo_egg-0.0.1-py3.6.egg"
+    expected_problem_message = "Could not locate import: thingy"
+    egg_file = Path(__file__).parent / "../../unit/source_code/samples/distribution/dist/thingy-0.0.1-py3.10.egg"
 
     entrypoint = make_directory()
 
@@ -239,7 +239,7 @@ def test_workflow_linter_lints_job_with_egg_dependency(
         ws.workspace.upload(remote_egg_file, f.read(), format=ImportFormat.AUTO)
     library = compute.Library(egg=remote_egg_file)
 
-    notebook = make_notebook(path=f"{entrypoint}/notebook.ipynb", content=b"import pkgdir")
+    notebook = make_notebook(path=f"{entrypoint}/notebook.ipynb", content=b"import thingy")
     job_with_egg_dependency = make_job(notebook_path=notebook, libraries=[library])
 
     problems = simple_ctx.workflow_linter.lint_job(job_with_egg_dependency.job_id)

tests/integration/source_code/test_jobs.py

@@ -85,15 +85,15 @@ def test_pip_cell_build_dependency_graph_resolves_installed_library(mock_path_lo
     dependency_resolver = DependencyResolver(pip_resolver, notebook_resolver, import_resolver, mock_path_lookup)
     graph = DependencyGraph(dependency, None, dependency_resolver, mock_path_lookup)
 
-    whl = Path(__file__).parent / '../samples/library-wheel/demo_egg-0.0.1-py3-none-any.whl'
+    whl = Path(__file__).parent / '../samples/distribution/dist/thingy-0.0.1-py2.py3-none-any.whl'
 
-    code = f"%pip install {whl.as_posix()}"  # installs pkgdir
+    code = f"%pip install {whl.as_posix()}"  # installs thingy
     cell = PipCell(code)
 
     problems = cell.build_dependency_graph(graph)
 
     assert len(problems) == 0
-    assert graph.path_lookup.resolve(Path("pkgdir")).exists()
+    assert graph.path_lookup.resolve(Path("thingy")).exists()
 
 
 def test_pip_cell_build_dependency_graph_handles_multiline_code():

tests/unit/source_code/notebooks/test_cells.py

@@ -0,0 +1,19 @@
+dist: pyproject.toml thingy/__init__.py thingy/__main__.py setup.py Makefile
+	hatch build
+
+dist/thingy-0.0.1.tar.gz: dist
+	@:
+
+dist/thingy-0.0.1-py3-none-any.whl: dist
+	@:
+
+dist/thingy-0.0.1-py3.10.egg: Makefile
+	python setup.py bdist_egg
+	git add dist/thingy-0.0.1-py3.10.egg -f
+	rm -fr build thingy.egg-info
+
+all: dist/thingy-0.0.1.tar.gz dist/thingy-0.0.1-py3-none-any.whl dist/thingy-0.0.1-py3.10.egg
+	@:
+
+clean:
+	rm -fr dist build thingy.egg-info

tests/unit/source_code/samples/distribution/Makefile

@@ -0,0 +1,6 @@
+# Demo wheel
+
+This contains sources for a wheel and an egg.
+
+Use `make all` to rebuild the wheel and egg after changing any content in this folder.
+

tests/unit/source_code/samples/distribution/README.md

@@ -0,0 +1,11 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "thingy"
+version = "0.0.1"
+description = 'This package does nothing useful and is meant for testing code distributions.'
+
+[project.entry-points.databricks]
+runtime = "thingy:main"

tests/unit/source_code/samples/distribution/dist/thingy-0.0.1-py2.py3-none-any.whl

@@ -0,0 +1,7 @@
+from setuptools import setup
+
+setup(
+    name="thingy",
+    version="0.0.1",
+    description='This package does nothing useful and is meant for testing eggs.',
+)