platformd: use file-based instead of abstract unix socket

.github/workflows/test-functional-platformd.yml

@@ -12,4 +12,5 @@ jobs:
       - uses: actions/setup-go@v5
         with:
           go-version: 1.26
+      - run: sudo groupadd platformd -g 9012 && sudo useradd platformd -u 9012 -g 9012
       - run: make functests-platformd

Makefile

@@ -108,6 +108,8 @@ functests-database:
 functests-platformd:
 	$(RUN) $(SUDO) FUNCTESTS_ENVOY_IMAGE=docker.io/envoyproxy/envoy:v1.31.4 \
                    FUNCTESTS_ENVOY_CONFIG=../../../dev/platformd/envoy-xds.yaml \
+                   FUNCTESTS_PLATFORMD_UID=9012 \
+                   FUNCTESTS_PLATFORMD_GID=9012 \
 				   go test -v ./test/functional/platformd $(ARGS)
 
 .PHONY: functests-shared

cmd/platformd/main.go

@@ -21,7 +21,9 @@ func main() {
 	var (
 		logger                       = slog.New(slog.NewTextHandler(os.Stdout, nil))
 		fs                           = flag.NewFlagSet("platformd", flag.ContinueOnError)
-		proxyServiceListenSock       = fs.String("management-server-listen-sock", "/var/run/platformd/platformd.sock", "path to the unix domain socket to listen on")              //nolint:lll
+		proxyServiceListenSock       = fs.String("management-server-listen-sock", "/run/platformd/platformd.sock", "path to the unix domain socket to listen on") //nolint:lll
+		mgmtSockUID                  = fs.Uint64("management-server-listen-sock-uid", 9012, "unix domain socket uid")
+		mgmtSockGID                  = fs.Uint64("management-server-listen-sock-gid", 9012, "unix domain socket gid")
 		criListenSock                = fs.String("cri-listen-sock", "/var/run/crio/crio.sock", "path to the unix domain socket the CRI is listening on")                           //nolint:lll
 		envoyImage                   = fs.String("envoy-image", "", "container image to use for envoy")                                                                            //nolint:lll
 		coreDNSImage                 = fs.String("coredns-image", "", "container image to use for CoreDNS")                                                                        //nolint:lll
@@ -89,6 +91,8 @@ func main() {
 				ContainerReadyTimeout:    *checkContainerReadyTimeout,
 				WaitAfterServerInit:      *checkWaitAfterServerInit,
 			},
+			ManagementSocketUID: *mgmtSockUID,
+			ManagementSocketGID: *mgmtSockGID,
 		}
 		ctx    = context.Background()
 		server = platformd.NewServer(logger)

dev/cni/00-netglue.conflist

@@ -5,7 +5,7 @@
     {
       "type": "netglue",
       "hostIface": "eth0",
-      "platformdListenSock": "unix-abstract:/run/platformd/platformd.sock",
+      "platformdListenSock": "/run/platformd/platformd.sock",
       "ipam": {
         "type": "host-local",
         "ranges":[

dev/platformd/config.json

@@ -1,5 +1,7 @@
 {
-  "management-server-listen-sock":  "@/run/platformd/platformd.sock",
+  "management-server-listen-sock":  "/run/platformd/platformd.sock",
+  "management-server-listen-sock-uid": 9012,
+  "management-server-listen-sock-gid": 9012,
   "cri-listen-sock": "unix:///var/run/crio/crio.sock",
   "getsockopt-cgroup": "/sys/fs/cgroup",
   "dns-server": "127.0.0.1:1053",

dev/platformd/envoy-xds.yaml

@@ -30,7 +30,7 @@ static_resources:
         - endpoint:
             address:
               pipe:
-                path: '@/run/platformd/platformd.sock'
+                path: 'run/platformd/platformd.sock'
     # It is recommended to configure either HTTP/2 or TCP keepalives in order to detect
     # connection issues, and allow Envoy to reconnect. TCP keepalive is less expensive, but
     # may be inadequate if there is a TCP proxy between Envoy and the management server.

hack/xcomp.yaml

@@ -9,6 +9,8 @@ images:
 mounts:
   - location: "/Users/yannic/proj/spc/explorer"
     writable: true
+  - location: "/Users/yannic/proj/spc/worktrees/explorer"
+    writable: true
 mountType: "virtiofs"
 portForwards:
   - guestPort: 3011

platformd/config.go

@@ -35,4 +35,6 @@ type Config struct {
 		ContainerReadyTimeout    time.Duration
 		WaitAfterServerInit      time.Duration
 	}
+	ManagementSocketUID uint64
+	ManagementSocketGID uint64
 }

platformd/server.go

@@ -2,11 +2,13 @@ package platformd
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"log/slog"
 	"net"
 	"net/netip"
 	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/envoyproxy/go-control-plane/pkg/cache/v3"
@@ -15,6 +17,7 @@ import (
 	"github.com/spacechunks/explorer/internal/image"
 	"github.com/spacechunks/explorer/platformd/garbage"
 	"github.com/spacechunks/explorer/platformd/status"
+	"google.golang.org/grpc/credentials"
 	"k8s.io/client-go/tools/remotecommand"
 
 	//instancev1alpha1 "github.com/spacechunks/explorer/api/instance/v1alpha1"
@@ -47,17 +50,21 @@ func NewServer(logger *slog.Logger) *Server {
 func (s *Server) Run(ctx context.Context, cfg Config) error {
 	s.logger.Info("started with config", "config", cfg)
 
-	//tlsCreds := credentials.NewTLS(&tls.Config{
-	//	InsecureSkipVerify: true,
-	//})
+	if err := os.MkdirAll(filepath.Dir(cfg.ManagementServerListenSock), os.ModePerm); err != nil {
+		return fmt.Errorf("create dir: %w", err)
+	}
+
+	tlsCreds := credentials.NewTLS(&tls.Config{
+		InsecureSkipVerify: true,
+	})
 
 	criConn, err := grpc.NewClient(cfg.CRIListenSock, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	if err != nil {
 		return fmt.Errorf("failed to create cri grpc client: %w", err)
 	}
 
-	//cpConn, err := grpc.NewClient(cfg.ControlPlaneEndpoint, grpc.WithTransportCredentials(tlsCreds))
-	cpConn, err := grpc.NewClient(cfg.ControlPlaneEndpoint, grpc.WithTransportCredentials(insecure.NewCredentials()))
+	cpConn, err := grpc.NewClient(cfg.ControlPlaneEndpoint, grpc.WithTransportCredentials(tlsCreds))
+	//cpConn, err := grpc.NewClient(cfg.ControlPlaneEndpoint, grpc.WithTransportCredentials(insecure.NewCredentials()))
 	if err != nil {
 		return fmt.Errorf("failed to create cri grpc client: %w", err)
 	}
@@ -290,6 +297,15 @@ func (s *Server) Run(ctx context.Context, cfg Config) error {
 			return fmt.Errorf("failed to listen on unix socket: %v", err)
 		}
 
+		if err := os.Chown(
+			cfg.ManagementServerListenSock,
+			int(cfg.ManagementSocketUID),
+			int(cfg.ManagementSocketGID),
+		); err != nil {
+			s.stopCh <- struct{}{}
+			return fmt.Errorf("failed to chown mgmt server socket: %w", err)
+		}
+
 		if err := mgmtServer.Serve(unixSock); err != nil {
 			s.stopCh <- struct{}{}
 			return fmt.Errorf("failed to serve mgmt server: %w", err)

test/fixture/proxy.go

@@ -19,12 +19,13 @@
 package fixture
 
 import (
+	"bytes"
 	"context"
+	"fmt"
 	"log/slog"
 	"net"
 	"net/netip"
 	"os"
-	"path/filepath"
 	"testing"
 	"time"
 
@@ -44,6 +45,57 @@ import (
 var (
 	EnvoyAdminAddr = "127.0.0.1:5555"
 	DNSUpstream    = netip.MustParseAddrPort("127.0.0.1:53")
+	envoyCfg       = `
+node:
+  cluster: system
+  id: proxy-0
+
+admin:
+  profile_path: /tmp/envoy.prof
+  address:
+    socket_address:
+      address: 127.0.0.1
+      port_value: 5555
+
+dynamic_resources:
+  ads_config:
+    api_type: GRPC
+    grpc_services:
+    - envoy_grpc:
+        cluster_name: ads
+  cds_config:
+    ads: {}
+  lds_config:
+    ads: {}
+
+static_resources:
+  clusters:
+  - name: ads
+    load_assignment:
+      cluster_name: ads
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              pipe:
+                path: '/tmp/platformd.sock'
+                mode: 0777
+    # It is recommended to configure either HTTP/2 or TCP keepalives in order to detect
+    # connection issues, and allow Envoy to reconnect. TCP keepalive is less expensive, but
+    # may be inadequate if there is a TCP proxy between Envoy and the management server.
+    # HTTP/2 keepalive is slightly more expensive, but may detect issues through more types
+    # of intermediate proxies.
+    typed_extension_protocol_options:
+      envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+        "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
+        explicit_http_config:
+          http2_protocol_options:
+            connection_keepalive:
+              interval: 30s
+              timeout: 5s
+    upstream_connection_options:
+      tcp_keepalive: {}
+`
 )
 
 func RunProxyAPIFixtures(ctx context.Context, t *testing.T) {
@@ -59,52 +111,80 @@ func RunProxyAPIFixtures(ctx context.Context, t *testing.T) {
 		)
 		proxyServ  = proxy.NewServer(svc)
 		envoyImage = os.Getenv("FUNCTESTS_ENVOY_IMAGE")
-		envoyCfg   = os.Getenv("FUNCTESTS_ENVOY_CONFIG")
 	)
 
-	path, err := filepath.Abs(envoyCfg)
+	proxyv1alpha1.RegisterProxyServiceServer(grpcServ, proxyServ)
+	xds.CreateAndRegisterServer(context.Background(), logger, grpcServ, ca)
+
+	require.NoError(t, svc.ApplyGlobalResources(ctx))
+
+	unixSock, err := net.Listen("unix", platformdAddr)
 	require.NoError(t, err)
 
+	err = os.Chown(platformdAddr, 9012, 9012)
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		grpcServ.Stop()
+		unixSock.Close()
+	})
+	go func() {
+		require.NoError(t, grpcServ.Serve(unixSock))
+	}()
+
+	test.WaitServerReady(t, "unix", platformdAddr, 20*time.Second)
+
+	// unix socket must be created, before we start envoy, otherwise we cannot mount it.
+
 	req := testcontainers.ContainerRequest{
 		Image: envoyImage,
+		ConfigModifier: func(cfg *container.Config) {
+			cfg.User = fmt.Sprintf(
+				"%s:%s",
+				os.Getenv("FUNCTESTS_PLATFORMD_UID"),
+				os.Getenv("FUNCTESTS_PLATFORMD_GID"),
+			)
+		},
 		Cmd: []string{
 			"-c", "/etc/envoy/config.yaml",
 		},
+		Files: []testcontainers.ContainerFile{
+			{
+				Reader:            bytes.NewReader([]byte(envoyCfg)),
+				ContainerFilePath: "/etc/envoy/config.yaml",
+				FileMode:          0777,
+			},
+		},
 		HostConfigModifier: func(cfg *container.HostConfig) {
 			cfg.NetworkMode = "host"
 			cfg.AutoRemove = true
 			cfg.Mounts = []mount.Mount{
 				{
 					Type:   mount.TypeBind,
-					Source: path,
-					Target: "/etc/envoy/config.yaml",
+					Source: "/tmp/platformd.sock",
+					Target: "/tmp/platformd.sock",
 				},
 			}
 		},
+		LogConsumerCfg: &testcontainers.LogConsumerConfig{
+			Consumers: []testcontainers.LogConsumer{
+				&testcontainers.StdoutLogConsumer{},
+			},
+		},
 	}
 
-	_, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+	ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
 		ContainerRequest: req,
 		Started:          true,
 	})
 	require.NoError(t, err)
 
-	test.WaitServerReady(t, "tcp", EnvoyAdminAddr, 20*time.Second)
-
-	proxyv1alpha1.RegisterProxyServiceServer(grpcServ, proxyServ)
-	xds.CreateAndRegisterServer(context.Background(), logger, grpcServ, ca)
-
-	require.NoError(t, svc.ApplyGlobalResources(ctx))
-
-	unixSock, err := net.Listen("unix", "@"+platformdAddr)
-	require.NoError(t, err)
 	t.Cleanup(func() {
-		grpcServ.Stop()
-		unixSock.Close()
+		// we need to terminate the container, because multiple envoy containers cause problems
+		if err := testcontainers.TerminateContainer(ctr); err != nil {
+			t.Logf("cannot terminate container: %v", err)
+		}
 	})
-	go func() {
-		require.NoError(t, grpcServ.Serve(unixSock))
-	}()
 
-	test.WaitServerReady(t, "unix", "@"+platformdAddr, 20*time.Second)
+	test.WaitServerReady(t, "tcp", EnvoyAdminAddr, 20*time.Second)
 }

test/fixture/shared.go

@@ -28,14 +28,11 @@ import (
 	_ "github.com/amacneil/dbmate/v2/pkg/driver/postgres"
 )
 
-// grpc client does not accept @ as abstract socket identifier,
-// so do not include it in the address string.
-
-const platformdAddr = "/run/platformd/platformd.sock"
+const platformdAddr = "/tmp/platformd.sock"
 
 func PlatformdClientConn(t *testing.T) *grpc.ClientConn {
 	conn, err := grpc.NewClient(
-		"unix-abstract:"+platformdAddr,
+		"unix://"+platformdAddr,
 		grpc.WithTransportCredentials(insecure.NewCredentials()),
 	)
 	require.NoError(t, err)

test/fixture/workload.go

@@ -45,7 +45,7 @@ func RunWorkloadAPIFixtures(t *testing.T) {
 		criServ = grpc.NewServer(grpc.Creds(insecure.NewCredentials()))
 	)
 
-	criSock, err := net.Listen("unix", "@"+platformdAddr)
+	criSock, err := net.Listen("unix", platformdAddr)
 	require.NoError(t, err)
 
 	cri := newFakeCRI()

test/functional/platformd/platformd_proxy_api_test.go

@@ -43,13 +43,12 @@ import (
 )
 
 func TestCreateListener(t *testing.T) {
-	var (
-		ctx = context.Background()
-		c   = proxyv1alpha1.NewProxyServiceClient(fixture.PlatformdClientConn(t))
-	)
+	ctx := context.Background()
 
 	fixture.RunProxyAPIFixtures(ctx, t)
 
+	c := proxyv1alpha1.NewProxyServiceClient(fixture.PlatformdClientConn(t))
+
 	var (
 		// ips used here have to be present on some interface on the machine the
 		// tests are being executed.
@@ -87,14 +86,14 @@ func TestCreateListener(t *testing.T) {
 }
 
 func TestDeleteListener(t *testing.T) {
+	ctx := context.Background()
+	fixture.RunProxyAPIFixtures(ctx, t)
+
 	var (
-		ctx  = context.Background()
 		wlID = "abc"
 		c    = proxyv1alpha1.NewProxyServiceClient(fixture.PlatformdClientConn(t))
 	)
 
-	fixture.RunProxyAPIFixtures(ctx, t)
-
 	createListeners(t, ctx, wlID, "127.0.0.1", c)
 
 	_, err := c.DeleteListeners(ctx, &proxyv1alpha1.DeleteListenersRequest{