From a109905cb21c21573efbc5750a12025bde6bdb9d Mon Sep 17 00:00:00 2001 From: Yi-Jyun Pan Date: Mon, 6 Nov 2023 17:34:20 +0800 Subject: [PATCH 01/10] fix(planner/python): RUN echo "skip install" --- internal/python/plan.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/python/plan.go b/internal/python/plan.go index cb81648b..5b993a79 100644 --- a/internal/python/plan.go +++ b/internal/python/plan.go @@ -492,7 +492,7 @@ func determineInstallCmd(ctx *pythonPlanContext) string { if command != "" { return command } - return "echo \"skip install\"" + return "RUN echo \"skip install\"" } func determineAptDependencies(ctx *pythonPlanContext) []string { From f1581d24a09a5c2d285122f963f4e68e679d3265 Mon Sep 17 00:00:00 2001 From: Yi-Jyun Pan Date: Mon, 6 Nov 2023 17:34:58 +0800 Subject: [PATCH 02/10] test(planner/python): Update snapshot --- internal/python/__snapshots__/plan_test.snap | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/python/__snapshots__/plan_test.snap b/internal/python/__snapshots__/plan_test.snap index b7ff90e9..7ca46a6d 100755 --- a/internal/python/__snapshots__/plan_test.snap +++ b/internal/python/__snapshots__/plan_test.snap @@ -65,7 +65,7 @@ RUN pip install uvicorn --- [TestDetermineInstallCmd_Snapshot/unknown-none - 1] -echo "skip install" +RUN echo "skip install" --- [TestDetermineStartCmd_Snapshot/pipenv-with-wsgi - 1] From c3321b89b6b9a2015d3942580da76c381f1d7693 Mon Sep 17 00:00:00 2001 From: Yi-Jyun Pan Date: Mon, 6 Nov 2023 17:42:07 +0800 Subject: [PATCH 03/10] feat(planner/python): Inject Config to planner --- internal/python/identify.go | 2 +- internal/python/plan.go | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/internal/python/identify.go b/internal/python/identify.go index 854b7e4e..4b7102d0 100644 --- a/internal/python/identify.go +++ b/internal/python/identify.go @@ -27,7 +27,7 @@ func (i *identify) Match(fs afero.Fs) bool { } func (i *identify) PlanMeta(options plan.NewPlannerOptions) types.PlanMeta { - return GetMeta(GetMetaOptions{Src: options.Source}) + return GetMeta(GetMetaOptions{Src: options.Source, Config: options.Config}) } var _ plan.Identifier = (*identify)(nil) diff --git a/internal/python/plan.go b/internal/python/plan.go index 5b993a79..5071a433 100644 --- a/internal/python/plan.go +++ b/internal/python/plan.go @@ -12,11 +12,13 @@ import ( "github.com/moznion/go-optional" "github.com/spf13/afero" "github.com/zeabur/zbpack/internal/utils" + "github.com/zeabur/zbpack/pkg/plan" "github.com/zeabur/zbpack/pkg/types" ) type pythonPlanContext struct { Src afero.Fs + Config plan.ImmutableProjectConfiguration PackageManager optional.Option[types.PackageManager] Framework optional.Option[types.PythonFramework] Entry optional.Option[string] @@ -635,7 +637,8 @@ func determineBuildCmd(ctx *pythonPlanContext) string { // GetMetaOptions is the options for GetMeta. type GetMetaOptions struct { - Src afero.Fs + Src afero.Fs + Config plan.ImmutableProjectConfiguration } // GetMeta returns the metadata of a Python project. @@ -643,7 +646,8 @@ func GetMeta(opt GetMetaOptions) types.PlanMeta { meta := types.PlanMeta{} ctx := &pythonPlanContext{ - Src: opt.Src, + Src: opt.Src, + Config: opt.Config, } pm := DeterminePackageManager(ctx) From c02794518f07942d6226342cabe6f9f0e36241a6 Mon Sep 17 00:00:00 2001 From: Yi-Jyun Pan Date: Mon, 6 Nov 2023 17:43:34 +0800 Subject: [PATCH 04/10] feat(planner/python): Implement streamlit support --- internal/python/identify.go | 2 +- internal/python/plan.go | 30 +++++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/internal/python/identify.go b/internal/python/identify.go index 4b7102d0..a6ff3b77 100644 --- a/internal/python/identify.go +++ b/internal/python/identify.go @@ -22,7 +22,7 @@ func (i *identify) PlanType() types.PlanType { func (i *identify) Match(fs afero.Fs) bool { return utils.HasFile( fs, - "app.py", "main.py", "app.py", "manage.py", "requirements.txt", + "app.py", "main.py", "app.py", "manage.py", "requirements.txt", "streamlit_app.py", ) } diff --git a/internal/python/plan.go b/internal/python/plan.go index 5071a433..be42fa16 100644 --- a/internal/python/plan.go +++ b/internal/python/plan.go @@ -2,6 +2,7 @@ package python import ( + "bytes" "errors" "fmt" "log" @@ -11,6 +12,7 @@ import ( "github.com/moznion/go-optional" "github.com/spf13/afero" + "github.com/spf13/cast" "github.com/zeabur/zbpack/internal/utils" "github.com/zeabur/zbpack/pkg/plan" "github.com/zeabur/zbpack/pkg/types" @@ -26,6 +28,12 @@ type pythonPlanContext struct { Static optional.Option[StaticInfo] } +const ( + // ConfigStreamlitEntry is the key for specifying the streamlit entry explicitly + // in the project configuration. + ConfigStreamlitEntry = "streamlit.entry" +) + // DetermineFramework determines the framework of the Python project. func DetermineFramework(ctx *pythonPlanContext) types.PythonFramework { src := ctx.Src @@ -547,7 +555,9 @@ func determineStartCmd(ctx *pythonPlanContext) string { commandSegment = append(commandSegment, "pdm run") } - if wsgi != "" { + if streamlitEntry := determineStreamlitEntry(ctx); streamlitEntry != "" { + commandSegment = append(commandSegment, "streamlit run", streamlitEntry) + } else if wsgi != "" { wsgilistenedPort := "8080" // The WSGI application should listen at 8000 @@ -635,6 +645,24 @@ func determineBuildCmd(ctx *pythonPlanContext) string { return "" } +func determineStreamlitEntry(ctx *pythonPlanContext) string { + src := ctx.Src + config := ctx.Config + + if streamlitEntry := plan.Cast(config.Get(ConfigStreamlitEntry), cast.ToStringE); streamlitEntry.IsSome() { + return streamlitEntry.Unwrap() + } + + for _, file := range []string{"app.py", "main.py", "streamlit_app.py"} { + content, err := afero.ReadFile(src, file) + if err == nil && bytes.Contains(content, []byte("import streamlit")) { + return file + } + } + + return "" +} + // GetMetaOptions is the options for GetMeta. type GetMetaOptions struct { Src afero.Fs From 871933a8232233162c560518728ccee0f3e0d85b Mon Sep 17 00:00:00 2001 From: Yi-Jyun Pan Date: Mon, 6 Nov 2023 17:44:04 +0800 Subject: [PATCH 05/10] test(planner/python): Test streamlit entry utility --- internal/python/plan_test.go | 62 ++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/internal/python/plan_test.go b/internal/python/plan_test.go index 2ae23a46..a3cbb760 100644 --- a/internal/python/plan_test.go +++ b/internal/python/plan_test.go @@ -9,6 +9,7 @@ import ( "github.com/moznion/go-optional" "github.com/spf13/afero" "github.com/stretchr/testify/assert" + "github.com/zeabur/zbpack/pkg/plan" "github.com/zeabur/zbpack/pkg/types" ) @@ -283,7 +284,11 @@ func TestDetermineStartCmd_Snapshot(t *testing.T) { t.Run(string(pm)+"-"+mode, func(t *testing.T) { t.Parallel() + fs := afero.NewMemMapFs() + ctx := pythonPlanContext{ + Src: fs, + Config: plan.NewProjectConfigurationFromFs(fs, ""), PackageManager: optional.Some(pm), Entry: optional.Some("app.py"), } @@ -726,3 +731,60 @@ func TestHasDependencyWithFile_Unknown(t *testing.T) { assert.False(t, HasDependencyWithFile(ctx, "flask")) assert.False(t, HasDependencyWithFile(ctx, "bar")) } + +func TestDetermineStreamlitEntry_ByFile(t *testing.T) { + fs := afero.NewMemMapFs() + _ = afero.WriteFile(fs, "streamlit_app.py", []byte(`import streamlit as st +x = st.slider("Select a value") +st.write(x, "squared is", x * x)`), 0o644) + _ = afero.WriteFile(fs, "requirements.txt", []byte("streamlit"), 0o644) + + config := plan.NewProjectConfigurationFromFs(fs, "") + + ctx := &pythonPlanContext{ + Src: fs, + Config: config, + PackageManager: optional.Some(types.PythonPackageManagerUnknown), + } + + assert.Equal(t, "streamlit_app.py", determineStreamlitEntry(ctx)) +} + +func TestDetermineStreamlitEntry_ByConfig(t *testing.T) { + fs := afero.NewMemMapFs() + _ = afero.WriteFile(fs, "zeabur_streamlit_demo.py", []byte(`import streamlit as st +x = st.slider("Select a value") +st.write(x, "squared is", x * x)`), 0o644) + _ = afero.WriteFile(fs, "requirements.txt", []byte("streamlit"), 0o644) + _ = afero.WriteFile(fs, "zbpack.json", []byte(`{"streamlit": {"entry": "zeabur_streamlit_demo.py"}}`), 0o644) + + config := plan.NewProjectConfigurationFromFs(fs, "") + + ctx := &pythonPlanContext{ + Src: fs, + Config: config, + PackageManager: optional.Some(types.PythonPackageManagerUnknown), + } + + assert.Equal(t, "zeabur_streamlit_demo.py", determineStreamlitEntry(ctx)) +} + +func TestDetermineStreamlitEntry_ConfigPrecedeConvention(t *testing.T) { + fs := afero.NewMemMapFs() + _ = afero.WriteFile(fs, "zeabur_streamlit_demo.py", []byte(`import streamlit as st +x = st.slider("Select a value") +st.write(x, "squared is", x * x)`), 0o644) + _ = afero.WriteFile(fs, "app.py", []byte(`print('not me')`), 0o644) + _ = afero.WriteFile(fs, "requirements.txt", []byte("streamlit"), 0o644) + _ = afero.WriteFile(fs, "zbpack.json", []byte(`{"streamlit": {"entry": "zeabur_streamlit_demo.py"}}`), 0o644) + + config := plan.NewProjectConfigurationFromFs(fs, "") + + ctx := &pythonPlanContext{ + Src: fs, + Config: config, + PackageManager: optional.Some(types.PythonPackageManagerUnknown), + } + + assert.Equal(t, "zeabur_streamlit_demo.py", determineStreamlitEntry(ctx)) +} From 41b2b4c179c62d3c5240d824f461e2d459dacd40 Mon Sep 17 00:00:00 2001 From: Yi-Jyun Pan Date: Mon, 6 Nov 2023 17:49:40 +0800 Subject: [PATCH 06/10] feat(planner/python): Cache StreamlitEntry result --- internal/python/plan.go | 15 ++++++++++++--- internal/python/plan_test.go | 21 +++++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/internal/python/plan.go b/internal/python/plan.go index be42fa16..acf38ae1 100644 --- a/internal/python/plan.go +++ b/internal/python/plan.go @@ -26,6 +26,7 @@ type pythonPlanContext struct { Entry optional.Option[string] Wsgi optional.Option[string] Static optional.Option[StaticInfo] + StreamlitEntry optional.Option[string] } const ( @@ -648,19 +649,27 @@ func determineBuildCmd(ctx *pythonPlanContext) string { func determineStreamlitEntry(ctx *pythonPlanContext) string { src := ctx.Src config := ctx.Config + se := &ctx.StreamlitEntry + + if entry, err := se.Take(); err == nil { + return entry + } if streamlitEntry := plan.Cast(config.Get(ConfigStreamlitEntry), cast.ToStringE); streamlitEntry.IsSome() { - return streamlitEntry.Unwrap() + *se = optional.Some(streamlitEntry.Unwrap()) + return se.Unwrap() } for _, file := range []string{"app.py", "main.py", "streamlit_app.py"} { content, err := afero.ReadFile(src, file) if err == nil && bytes.Contains(content, []byte("import streamlit")) { - return file + *se = optional.Some(file) + return se.Unwrap() } } - return "" + *se = optional.Some("") + return se.Unwrap() } // GetMetaOptions is the options for GetMeta. diff --git a/internal/python/plan_test.go b/internal/python/plan_test.go index a3cbb760..61a269e5 100644 --- a/internal/python/plan_test.go +++ b/internal/python/plan_test.go @@ -788,3 +788,24 @@ st.write(x, "squared is", x * x)`), 0o644) assert.Equal(t, "zeabur_streamlit_demo.py", determineStreamlitEntry(ctx)) } + +func TestDetermineStreamlitEntry_Cache(t *testing.T) { + fs := afero.NewMemMapFs() + _ = afero.WriteFile(fs, "zeabur_streamlit_demo.py", []byte(`import streamlit as st +x = st.slider("Select a value") +st.write(x, "squared is", x * x)`), 0o644) + _ = afero.WriteFile(fs, "app.py", []byte(`print('not me')`), 0o644) + _ = afero.WriteFile(fs, "requirements.txt", []byte("streamlit"), 0o644) + _ = afero.WriteFile(fs, "zbpack.json", []byte(`{"streamlit": {"entry": "zeabur_streamlit_demo.py"}}`), 0o644) + + config := plan.NewProjectConfigurationFromFs(fs, "") + + ctx := &pythonPlanContext{ + Src: fs, + Config: config, + PackageManager: optional.Some(types.PythonPackageManagerUnknown), + } + + assert.Equal(t, "zeabur_streamlit_demo.py", determineStreamlitEntry(ctx)) + assert.Equal(t, "zeabur_streamlit_demo.py", ctx.StreamlitEntry.Unwrap()) +} From 3bb834a9d561cafa39ba29942bd4dd3454f1a9b4 Mon Sep 17 00:00:00 2001 From: Yi-Jyun Pan Date: Mon, 6 Nov 2023 17:54:52 +0800 Subject: [PATCH 07/10] feat(planner/python): Install streamlit even no requirements.txt there --- internal/python/plan.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/internal/python/plan.go b/internal/python/plan.go index acf38ae1..84258288 100644 --- a/internal/python/plan.go +++ b/internal/python/plan.go @@ -458,6 +458,11 @@ func determineInstallCmd(ctx *pythonPlanContext) string { commands = append(commands, "RUN pipenv install gunicorn") } } + + if determineStreamlitEntry(ctx) != "" { + commands = append(commands, "RUN pipenv install streamlit") + } + commands = append(commands, "COPY Pipfile* .", "RUN pipenv install") case types.PythonPackageManagerPoetry: commands = append(commands, "RUN pip install poetry") @@ -469,9 +474,15 @@ func determineInstallCmd(ctx *pythonPlanContext) string { commands = append(commands, "RUN poetry add gunicorn") } } + + if determineStreamlitEntry(ctx) != "" { + commands = append(commands, "RUN poetry add streamlit") + } + commands = append(commands, "COPY poetry.lock* pyproject.toml* .", "RUN poetry install") case types.PythonPackageManagerPdm: commands = append(commands, "COPY pdm.lock* pyproject.toml* .", "RUN pip install pdm") + if wsgi != "" { if framework == types.PythonFrameworkFastapi { commands = append(commands, "RUN pdm add uvicorn") @@ -479,6 +490,11 @@ func determineInstallCmd(ctx *pythonPlanContext) string { commands = append(commands, "RUN pdm add gunicorn") } } + + if determineStreamlitEntry(ctx) != "" { + commands = append(commands, "RUN pdm add streamlit") + } + commands = append(commands, "RUN pdm install") case types.PythonPackageManagerPip: if wsgi != "" { @@ -488,6 +504,11 @@ func determineInstallCmd(ctx *pythonPlanContext) string { commands = append(commands, "RUN pip install gunicorn") } } + + if determineStreamlitEntry(ctx) != "" { + commands = append(commands, "RUN pip install streamlit") + } + commands = append(commands, "COPY requirements.txt* .", "RUN pip install -r requirements.txt") default: if wsgi != "" { @@ -497,6 +518,10 @@ func determineInstallCmd(ctx *pythonPlanContext) string { commands = append(commands, "RUN pip install gunicorn") } } + + if determineStreamlitEntry(ctx) != "" { + commands = append(commands, "RUN pip install streamlit") + } } command := strings.Join(commands, "\n") From 76b6684efc5c76abc411f69620627803d2bf2657 Mon Sep 17 00:00:00 2001 From: Yi-Jyun Pan Date: Mon, 6 Nov 2023 17:55:10 +0800 Subject: [PATCH 08/10] test(planner/python): Test StreamlitEntry --- internal/python/__snapshots__/plan_test.snap | 24 ++++++++++++++++++++ internal/python/plan_test.go | 11 +++++++++ 2 files changed, 35 insertions(+) diff --git a/internal/python/__snapshots__/plan_test.snap b/internal/python/__snapshots__/plan_test.snap index 7ca46a6d..f2ceb1cf 100755 --- a/internal/python/__snapshots__/plan_test.snap +++ b/internal/python/__snapshots__/plan_test.snap @@ -203,3 +203,27 @@ RUN pipenv install gunicorn COPY Pipfile* . RUN pipenv install --- + +[TestDetermineInstallCmd_Snapshot/pipenv-with-streamlit-entry - 1] +RUN pip install pipenv +RUN pipenv install streamlit +COPY Pipfile* . +RUN pipenv install +--- + +[TestDetermineInstallCmd_Snapshot/pip-with-streamlit-entry - 1] +RUN pip install streamlit +COPY requirements.txt* . +RUN pip install -r requirements.txt +--- + +[TestDetermineInstallCmd_Snapshot/poetry-with-streamlit-entry - 1] +RUN pip install poetry +RUN poetry add streamlit +COPY poetry.lock* pyproject.toml* . +RUN poetry install +--- + +[TestDetermineInstallCmd_Snapshot/unknown-with-streamlit-entry - 1] +RUN pip install streamlit +--- diff --git a/internal/python/plan_test.go b/internal/python/plan_test.go index 61a269e5..c3ba2811 100644 --- a/internal/python/plan_test.go +++ b/internal/python/plan_test.go @@ -190,6 +190,7 @@ func TestDetermineInstallCmd_Snapshot(t *testing.T) { WithStaticDjango = "with-static-django" WithStaticNginx = "with-static-nginx" WithStaticNginxDjango = "with-static-nginx-django" + WithStreamlitEntry = "with-streamlit-entry" None = "none" ) @@ -206,13 +207,19 @@ func TestDetermineInstallCmd_Snapshot(t *testing.T) { WithStaticNginx, WithStaticDjango, WithStaticNginxDjango, + WithStreamlitEntry, None, } { mode := mode t.Run(string(pm)+"-"+mode, func(t *testing.T) { t.Parallel() + fs := afero.NewMemMapFs() + config := plan.NewProjectConfigurationFromFs(fs, "") + ctx := pythonPlanContext{ + Src: fs, + Config: config, PackageManager: optional.Some(pm), } @@ -257,6 +264,10 @@ func TestDetermineInstallCmd_Snapshot(t *testing.T) { }) } + if mode == WithStreamlitEntry { + ctx.StreamlitEntry = optional.Some("streamlit_app.py") + } + ic := determineInstallCmd(&ctx) snaps.MatchSnapshot(t, ic) }) From 97954eb706441c03806f17a97fa7e2a510046c0d Mon Sep 17 00:00:00 2001 From: Yi-Jyun Pan Date: Mon, 6 Nov 2023 18:11:54 +0800 Subject: [PATCH 09/10] fix(planner/python): Specify host and port in streamlit --- internal/python/plan.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/python/plan.go b/internal/python/plan.go index 84258288..6a3fa0f0 100644 --- a/internal/python/plan.go +++ b/internal/python/plan.go @@ -582,7 +582,7 @@ func determineStartCmd(ctx *pythonPlanContext) string { } if streamlitEntry := determineStreamlitEntry(ctx); streamlitEntry != "" { - commandSegment = append(commandSegment, "streamlit run", streamlitEntry) + commandSegment = append(commandSegment, "streamlit run", streamlitEntry, "--server.port=8080", "--server.address=0.0.0.0") } else if wsgi != "" { wsgilistenedPort := "8080" From e1ca363c7e51c84a3329ecbc25d363f79972727c Mon Sep 17 00:00:00 2001 From: Yi-Jyun Pan Date: Mon, 6 Nov 2023 18:12:16 +0800 Subject: [PATCH 10/10] test(planner/python): Minimum Streamlit example --- tests/python-streamlit/streamlit_app.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/python-streamlit/streamlit_app.py diff --git a/tests/python-streamlit/streamlit_app.py b/tests/python-streamlit/streamlit_app.py new file mode 100644 index 00000000..ac9d89ab --- /dev/null +++ b/tests/python-streamlit/streamlit_app.py @@ -0,0 +1,3 @@ +import streamlit as st +x = st.slider("Select a value") +st.write(x, "squared is", x * x)