diff --git a/.github/workflows/test-functional-platformd.yml b/.github/workflows/test-functional-platformd.yml index e8cc2092..91f4e3e0 100644 --- a/.github/workflows/test-functional-platformd.yml +++ b/.github/workflows/test-functional-platformd.yml @@ -12,4 +12,5 @@ jobs: - uses: actions/setup-go@v5 with: go-version: 1.26 + - run: sudo groupadd platformd -g 9012 && sudo useradd platformd -u 9012 -g 9012 - run: make functests-platformd \ No newline at end of file diff --git a/Makefile b/Makefile index 8efd1826..7f69827d 100644 --- a/Makefile +++ b/Makefile @@ -108,6 +108,8 @@ functests-database: functests-platformd: $(RUN) $(SUDO) FUNCTESTS_ENVOY_IMAGE=docker.io/envoyproxy/envoy:v1.31.4 \ FUNCTESTS_ENVOY_CONFIG=../../../dev/platformd/envoy-xds.yaml \ + FUNCTESTS_PLATFORMD_UID=9012 \ + FUNCTESTS_PLATFORMD_GID=9012 \ go test -v ./test/functional/platformd $(ARGS) .PHONY: functests-shared diff --git a/cmd/platformd/main.go b/cmd/platformd/main.go index e544881b..34831b98 100644 --- a/cmd/platformd/main.go +++ b/cmd/platformd/main.go @@ -21,7 +21,9 @@ func main() { var ( logger = slog.New(slog.NewTextHandler(os.Stdout, nil)) fs = flag.NewFlagSet("platformd", flag.ContinueOnError) - proxyServiceListenSock = fs.String("management-server-listen-sock", "/var/run/platformd/platformd.sock", "path to the unix domain socket to listen on") //nolint:lll + proxyServiceListenSock = fs.String("management-server-listen-sock", "/run/platformd/platformd.sock", "path to the unix domain socket to listen on") //nolint:lll + mgmtSockUID = fs.Uint64("management-server-listen-sock-uid", 9012, "unix domain socket uid") + mgmtSockGID = fs.Uint64("management-server-listen-sock-gid", 9012, "unix domain socket gid") criListenSock = fs.String("cri-listen-sock", "/var/run/crio/crio.sock", "path to the unix domain socket the CRI is listening on") //nolint:lll envoyImage = fs.String("envoy-image", "", "container image to use for envoy") //nolint:lll coreDNSImage = fs.String("coredns-image", "", "container image to use for CoreDNS") //nolint:lll @@ -89,6 +91,8 @@ func main() { ContainerReadyTimeout: *checkContainerReadyTimeout, WaitAfterServerInit: *checkWaitAfterServerInit, }, + ManagementSocketUID: *mgmtSockUID, + ManagementSocketGID: *mgmtSockGID, } ctx = context.Background() server = platformd.NewServer(logger) diff --git a/dev/cni/00-netglue.conflist b/dev/cni/00-netglue.conflist index 05f1234b..78dc90b5 100644 --- a/dev/cni/00-netglue.conflist +++ b/dev/cni/00-netglue.conflist @@ -5,7 +5,7 @@ { "type": "netglue", "hostIface": "eth0", - "platformdListenSock": "unix-abstract:/run/platformd/platformd.sock", + "platformdListenSock": "/run/platformd/platformd.sock", "ipam": { "type": "host-local", "ranges":[ diff --git a/dev/platformd/config.json b/dev/platformd/config.json index b2b43175..7c8c72da 100644 --- a/dev/platformd/config.json +++ b/dev/platformd/config.json @@ -1,5 +1,7 @@ { - "management-server-listen-sock": "@/run/platformd/platformd.sock", + "management-server-listen-sock": "/run/platformd/platformd.sock", + "management-server-listen-sock-uid": 9012, + "management-server-listen-sock-gid": 9012, "cri-listen-sock": "unix:///var/run/crio/crio.sock", "getsockopt-cgroup": "/sys/fs/cgroup", "dns-server": "127.0.0.1:1053", diff --git a/dev/platformd/envoy-xds.yaml b/dev/platformd/envoy-xds.yaml index 1951c0fe..094efee5 100644 --- a/dev/platformd/envoy-xds.yaml +++ b/dev/platformd/envoy-xds.yaml @@ -30,7 +30,7 @@ static_resources: - endpoint: address: pipe: - path: '@/run/platformd/platformd.sock' + path: 'run/platformd/platformd.sock' # It is recommended to configure either HTTP/2 or TCP keepalives in order to detect # connection issues, and allow Envoy to reconnect. TCP keepalive is less expensive, but # may be inadequate if there is a TCP proxy between Envoy and the management server. diff --git a/hack/xcomp.yaml b/hack/xcomp.yaml index 06ab6d5d..9269f806 100644 --- a/hack/xcomp.yaml +++ b/hack/xcomp.yaml @@ -9,6 +9,8 @@ images: mounts: - location: "/Users/yannic/proj/spc/explorer" writable: true + - location: "/Users/yannic/proj/spc/worktrees/explorer" + writable: true mountType: "virtiofs" portForwards: - guestPort: 3011 diff --git a/platformd/config.go b/platformd/config.go index 08e17b09..5a84e998 100644 --- a/platformd/config.go +++ b/platformd/config.go @@ -35,4 +35,6 @@ type Config struct { ContainerReadyTimeout time.Duration WaitAfterServerInit time.Duration } + ManagementSocketUID uint64 + ManagementSocketGID uint64 } diff --git a/platformd/server.go b/platformd/server.go index 08e9d804..5147ec1a 100644 --- a/platformd/server.go +++ b/platformd/server.go @@ -2,11 +2,13 @@ package platformd import ( "context" + "crypto/tls" "fmt" "log/slog" "net" "net/netip" "os" + "path/filepath" "time" "github.com/envoyproxy/go-control-plane/pkg/cache/v3" @@ -15,6 +17,7 @@ import ( "github.com/spacechunks/explorer/internal/image" "github.com/spacechunks/explorer/platformd/garbage" "github.com/spacechunks/explorer/platformd/status" + "google.golang.org/grpc/credentials" "k8s.io/client-go/tools/remotecommand" //instancev1alpha1 "github.com/spacechunks/explorer/api/instance/v1alpha1" @@ -47,17 +50,21 @@ func NewServer(logger *slog.Logger) *Server { func (s *Server) Run(ctx context.Context, cfg Config) error { s.logger.Info("started with config", "config", cfg) - //tlsCreds := credentials.NewTLS(&tls.Config{ - // InsecureSkipVerify: true, - //}) + if err := os.MkdirAll(filepath.Dir(cfg.ManagementServerListenSock), os.ModePerm); err != nil { + return fmt.Errorf("create dir: %w", err) + } + + tlsCreds := credentials.NewTLS(&tls.Config{ + InsecureSkipVerify: true, + }) criConn, err := grpc.NewClient(cfg.CRIListenSock, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { return fmt.Errorf("failed to create cri grpc client: %w", err) } - //cpConn, err := grpc.NewClient(cfg.ControlPlaneEndpoint, grpc.WithTransportCredentials(tlsCreds)) - cpConn, err := grpc.NewClient(cfg.ControlPlaneEndpoint, grpc.WithTransportCredentials(insecure.NewCredentials())) + cpConn, err := grpc.NewClient(cfg.ControlPlaneEndpoint, grpc.WithTransportCredentials(tlsCreds)) + //cpConn, err := grpc.NewClient(cfg.ControlPlaneEndpoint, grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { return fmt.Errorf("failed to create cri grpc client: %w", err) } @@ -290,6 +297,15 @@ func (s *Server) Run(ctx context.Context, cfg Config) error { return fmt.Errorf("failed to listen on unix socket: %v", err) } + if err := os.Chown( + cfg.ManagementServerListenSock, + int(cfg.ManagementSocketUID), + int(cfg.ManagementSocketGID), + ); err != nil { + s.stopCh <- struct{}{} + return fmt.Errorf("failed to chown mgmt server socket: %w", err) + } + if err := mgmtServer.Serve(unixSock); err != nil { s.stopCh <- struct{}{} return fmt.Errorf("failed to serve mgmt server: %w", err) diff --git a/test/fixture/proxy.go b/test/fixture/proxy.go index 9759bb29..4a5cbd6c 100644 --- a/test/fixture/proxy.go +++ b/test/fixture/proxy.go @@ -19,12 +19,13 @@ package fixture import ( + "bytes" "context" + "fmt" "log/slog" "net" "net/netip" "os" - "path/filepath" "testing" "time" @@ -44,6 +45,57 @@ import ( var ( EnvoyAdminAddr = "127.0.0.1:5555" DNSUpstream = netip.MustParseAddrPort("127.0.0.1:53") + envoyCfg = ` +node: + cluster: system + id: proxy-0 + +admin: + profile_path: /tmp/envoy.prof + address: + socket_address: + address: 127.0.0.1 + port_value: 5555 + +dynamic_resources: + ads_config: + api_type: GRPC + grpc_services: + - envoy_grpc: + cluster_name: ads + cds_config: + ads: {} + lds_config: + ads: {} + +static_resources: + clusters: + - name: ads + load_assignment: + cluster_name: ads + endpoints: + - lb_endpoints: + - endpoint: + address: + pipe: + path: '/tmp/platformd.sock' + mode: 0777 + # It is recommended to configure either HTTP/2 or TCP keepalives in order to detect + # connection issues, and allow Envoy to reconnect. TCP keepalive is less expensive, but + # may be inadequate if there is a TCP proxy between Envoy and the management server. + # HTTP/2 keepalive is slightly more expensive, but may detect issues through more types + # of intermediate proxies. + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + upstream_connection_options: + tcp_keepalive: {} +` ) func RunProxyAPIFixtures(ctx context.Context, t *testing.T) { @@ -59,52 +111,80 @@ func RunProxyAPIFixtures(ctx context.Context, t *testing.T) { ) proxyServ = proxy.NewServer(svc) envoyImage = os.Getenv("FUNCTESTS_ENVOY_IMAGE") - envoyCfg = os.Getenv("FUNCTESTS_ENVOY_CONFIG") ) - path, err := filepath.Abs(envoyCfg) + proxyv1alpha1.RegisterProxyServiceServer(grpcServ, proxyServ) + xds.CreateAndRegisterServer(context.Background(), logger, grpcServ, ca) + + require.NoError(t, svc.ApplyGlobalResources(ctx)) + + unixSock, err := net.Listen("unix", platformdAddr) require.NoError(t, err) + err = os.Chown(platformdAddr, 9012, 9012) + require.NoError(t, err) + + t.Cleanup(func() { + grpcServ.Stop() + unixSock.Close() + }) + go func() { + require.NoError(t, grpcServ.Serve(unixSock)) + }() + + test.WaitServerReady(t, "unix", platformdAddr, 20*time.Second) + + // unix socket must be created, before we start envoy, otherwise we cannot mount it. + req := testcontainers.ContainerRequest{ Image: envoyImage, + ConfigModifier: func(cfg *container.Config) { + cfg.User = fmt.Sprintf( + "%s:%s", + os.Getenv("FUNCTESTS_PLATFORMD_UID"), + os.Getenv("FUNCTESTS_PLATFORMD_GID"), + ) + }, Cmd: []string{ "-c", "/etc/envoy/config.yaml", }, + Files: []testcontainers.ContainerFile{ + { + Reader: bytes.NewReader([]byte(envoyCfg)), + ContainerFilePath: "/etc/envoy/config.yaml", + FileMode: 0777, + }, + }, HostConfigModifier: func(cfg *container.HostConfig) { cfg.NetworkMode = "host" cfg.AutoRemove = true cfg.Mounts = []mount.Mount{ { Type: mount.TypeBind, - Source: path, - Target: "/etc/envoy/config.yaml", + Source: "/tmp/platformd.sock", + Target: "/tmp/platformd.sock", }, } }, + LogConsumerCfg: &testcontainers.LogConsumerConfig{ + Consumers: []testcontainers.LogConsumer{ + &testcontainers.StdoutLogConsumer{}, + }, + }, } - _, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ ContainerRequest: req, Started: true, }) require.NoError(t, err) - test.WaitServerReady(t, "tcp", EnvoyAdminAddr, 20*time.Second) - - proxyv1alpha1.RegisterProxyServiceServer(grpcServ, proxyServ) - xds.CreateAndRegisterServer(context.Background(), logger, grpcServ, ca) - - require.NoError(t, svc.ApplyGlobalResources(ctx)) - - unixSock, err := net.Listen("unix", "@"+platformdAddr) - require.NoError(t, err) t.Cleanup(func() { - grpcServ.Stop() - unixSock.Close() + // we need to terminate the container, because multiple envoy containers cause problems + if err := testcontainers.TerminateContainer(ctr); err != nil { + t.Logf("cannot terminate container: %v", err) + } }) - go func() { - require.NoError(t, grpcServ.Serve(unixSock)) - }() - test.WaitServerReady(t, "unix", "@"+platformdAddr, 20*time.Second) + test.WaitServerReady(t, "tcp", EnvoyAdminAddr, 20*time.Second) } diff --git a/test/fixture/shared.go b/test/fixture/shared.go index 655cf3db..0181b311 100644 --- a/test/fixture/shared.go +++ b/test/fixture/shared.go @@ -28,14 +28,11 @@ import ( _ "github.com/amacneil/dbmate/v2/pkg/driver/postgres" ) -// grpc client does not accept @ as abstract socket identifier, -// so do not include it in the address string. - -const platformdAddr = "/run/platformd/platformd.sock" +const platformdAddr = "/tmp/platformd.sock" func PlatformdClientConn(t *testing.T) *grpc.ClientConn { conn, err := grpc.NewClient( - "unix-abstract:"+platformdAddr, + "unix://"+platformdAddr, grpc.WithTransportCredentials(insecure.NewCredentials()), ) require.NoError(t, err) diff --git a/test/fixture/workload.go b/test/fixture/workload.go index 56e55212..a48aace2 100644 --- a/test/fixture/workload.go +++ b/test/fixture/workload.go @@ -45,7 +45,7 @@ func RunWorkloadAPIFixtures(t *testing.T) { criServ = grpc.NewServer(grpc.Creds(insecure.NewCredentials())) ) - criSock, err := net.Listen("unix", "@"+platformdAddr) + criSock, err := net.Listen("unix", platformdAddr) require.NoError(t, err) cri := newFakeCRI() diff --git a/test/functional/platformd/platformd_proxy_api_test.go b/test/functional/platformd/platformd_proxy_api_test.go index 5f919aa3..b8076ad8 100644 --- a/test/functional/platformd/platformd_proxy_api_test.go +++ b/test/functional/platformd/platformd_proxy_api_test.go @@ -43,13 +43,12 @@ import ( ) func TestCreateListener(t *testing.T) { - var ( - ctx = context.Background() - c = proxyv1alpha1.NewProxyServiceClient(fixture.PlatformdClientConn(t)) - ) + ctx := context.Background() fixture.RunProxyAPIFixtures(ctx, t) + c := proxyv1alpha1.NewProxyServiceClient(fixture.PlatformdClientConn(t)) + var ( // ips used here have to be present on some interface on the machine the // tests are being executed. @@ -87,14 +86,14 @@ func TestCreateListener(t *testing.T) { } func TestDeleteListener(t *testing.T) { + ctx := context.Background() + fixture.RunProxyAPIFixtures(ctx, t) + var ( - ctx = context.Background() wlID = "abc" c = proxyv1alpha1.NewProxyServiceClient(fixture.PlatformdClientConn(t)) ) - fixture.RunProxyAPIFixtures(ctx, t) - createListeners(t, ctx, wlID, "127.0.0.1", c) _, err := c.DeleteListeners(ctx, &proxyv1alpha1.DeleteListenersRequest{