From 4f354c75e8956ffa8434e8fd9e7cd1e7d4a2c7e5 Mon Sep 17 00:00:00 2001 From: John Cai Date: Tue, 4 Jun 2019 18:16:09 -0700 Subject: [PATCH] Add filesystem metadata file on startup Write a metadata file on startup with a unique id. Also add a field to the ServerInfoResponse message that returns this unique id as well as the path to the file on the server --- NOTICE | 29 +++++++ changelogs/unreleased/jc-filesystem-uuid.yml | 5 ++ cmd/gitaly/main.go | 7 ++ go.mod | 3 +- go.sum | 6 +- internal/service/server/info.go | 24 ++++-- internal/service/server/info_test.go | 12 +++ internal/storage/metadata.go | 67 ++++++++++++++++ internal/storage/metadata_test.go | 83 ++++++++++++++++++++ internal/storage/testdata/.gitaly-metadata | 1 + 10 files changed, 228 insertions(+), 9 deletions(-) create mode 100644 changelogs/unreleased/jc-filesystem-uuid.yml create mode 100644 internal/storage/metadata.go create mode 100644 internal/storage/metadata_test.go create mode 100644 internal/storage/testdata/.gitaly-metadata diff --git a/NOTICE b/NOTICE index 09c9718549..49b4bc2468 100644 --- a/NOTICE +++ b/NOTICE @@ -438,6 +438,35 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +LICENSE - gitlab.com/gitlab-org/gitaly/vendor/github.com/google/uuid +Copyright (c) 2009,2014 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ LICENSE - gitlab.com/gitlab-org/gitaly/vendor/github.com/grpc-ecosystem/go-grpc-middleware Apache License diff --git a/changelogs/unreleased/jc-filesystem-uuid.yml b/changelogs/unreleased/jc-filesystem-uuid.yml new file mode 100644 index 0000000000..5fd0244bb7 --- /dev/null +++ b/changelogs/unreleased/jc-filesystem-uuid.yml @@ -0,0 +1,5 @@ +--- +title: Add filesystem metadata file on startup +merge_request: 1289 +author: +type: other diff --git a/cmd/gitaly/main.go b/cmd/gitaly/main.go index fb87ecfe3e..f3f97413b9 100644 --- a/cmd/gitaly/main.go +++ b/cmd/gitaly/main.go @@ -13,6 +13,7 @@ import ( "gitlab.com/gitlab-org/gitaly/internal/config" "gitlab.com/gitlab-org/gitaly/internal/git" "gitlab.com/gitlab-org/gitaly/internal/server" + "gitlab.com/gitlab-org/gitaly/internal/storage" "gitlab.com/gitlab-org/gitaly/internal/tempdir" "gitlab.com/gitlab-org/gitaly/internal/version" "gitlab.com/gitlab-org/labkit/tracing" @@ -152,6 +153,12 @@ func run(b *bootstrap.Bootstrap) error { }) } + for _, shard := range config.Config.Storages { + if err = storage.WriteMetadataFile(shard); err != nil { + log.WithError(err).Error("Unable to write gitaly metadata file") + } + } + if err := b.Start(); err != nil { return fmt.Errorf("unable to start the bootstrap: %v", err) } diff --git a/go.mod b/go.mod index 0daf1038a7..2f8d016c16 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ require ( github.com/cloudflare/tableflip v0.0.0-20190329062924-8392f1641731 github.com/getsentry/raven-go v0.1.2 github.com/golang/protobuf v1.3.1 + github.com/google/uuid v1.1.1 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/kelseyhightower/envconfig v1.3.0 @@ -13,7 +14,7 @@ require ( github.com/sirupsen/logrus v1.2.0 github.com/stretchr/testify v1.2.2 github.com/tinylib/msgp v1.1.0 // indirect - gitlab.com/gitlab-org/gitaly-proto v1.32.0 + gitlab.com/gitlab-org/gitaly-proto v1.33.0 gitlab.com/gitlab-org/labkit v0.0.0-20190221122536-0c3fc7cdd57c golang.org/x/net v0.0.0-20190311183353-d8887717615a golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 diff --git a/go.sum b/go.sum index cc56b0adbd..70d35d346d 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= @@ -102,8 +104,8 @@ github.com/uber/jaeger-client-go v2.15.0+incompatible h1:NP3qsSqNxh8VYr956ur1N/1 github.com/uber/jaeger-client-go v2.15.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v1.5.0 h1:OHbgr8l656Ub3Fw5k9SWnBfIEwvoHQ+W2y+Aa9D1Uyo= github.com/uber/jaeger-lib v1.5.0/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -gitlab.com/gitlab-org/gitaly-proto v1.32.0 h1:TRe/iw/Gid1RNM2VzK+WICIw4/N7V5s0IdhmgiPyqNE= -gitlab.com/gitlab-org/gitaly-proto v1.32.0/go.mod h1:zNjk/86bjwLVJ4NcvInBcXcLdptdRFQ28sYrdFbrFgY= +gitlab.com/gitlab-org/gitaly-proto v1.33.0 h1:gSwV1hGpwrEauYcl81j214DRfUHAznBeeOMdwbvadnc= +gitlab.com/gitlab-org/gitaly-proto v1.33.0/go.mod h1:zNjk/86bjwLVJ4NcvInBcXcLdptdRFQ28sYrdFbrFgY= gitlab.com/gitlab-org/labkit v0.0.0-20190221122536-0c3fc7cdd57c h1:xo48LcGsTCasKcJpQDBCCuZU+aP8uGaboUVvD7Lgm6g= gitlab.com/gitlab-org/labkit v0.0.0-20190221122536-0c3fc7cdd57c/go.mod h1:rYhLgfrbEcyfinG+R3EvKu6bZSsmwQqcXzLfHWSfUKM= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= diff --git a/internal/service/server/info.go b/internal/service/server/info.go index 166eef9431..ca54cf5d02 100644 --- a/internal/service/server/info.go +++ b/internal/service/server/info.go @@ -6,35 +6,47 @@ import ( "os" "path" + grpc_logrus "github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus" "gitlab.com/gitlab-org/gitaly-proto/go/gitalypb" + "gitlab.com/gitlab-org/gitaly/internal/config" "gitlab.com/gitlab-org/gitaly/internal/git" + "gitlab.com/gitlab-org/gitaly/internal/helper" "gitlab.com/gitlab-org/gitaly/internal/helper/fstype" + "gitlab.com/gitlab-org/gitaly/internal/storage" "gitlab.com/gitlab-org/gitaly/internal/version" ) func (s *server) ServerInfo(ctx context.Context, in *gitalypb.ServerInfoRequest) (*gitalypb.ServerInfoResponse, error) { gitVersion, err := git.Version() + if err != nil { + return nil, helper.ErrInternal(err) + } var storageStatuses []*gitalypb.ServerInfoResponse_StorageStatus for _, shard := range config.Config.Storages { readable, writeable := shardCheck(shard.Path) fsType := fstype.FileSystem(shard.Path) + gitalyMetadata, err := storage.ReadMetadataFile(shard) + if err != nil { + grpc_logrus.Extract(ctx).WithField("storage", shard).WithError(err).Error("reading gitaly metadata file") + } + storageStatuses = append(storageStatuses, &gitalypb.ServerInfoResponse_StorageStatus{ - StorageName: shard.Name, - Readable: readable, - Writeable: writeable, - FsType: fsType, + StorageName: shard.Name, + Readable: readable, + Writeable: writeable, + FsType: fsType, + FilesystemId: gitalyMetadata.GitalyFilesystemID, }) - } return &gitalypb.ServerInfoResponse{ ServerVersion: version.GetVersion(), GitVersion: gitVersion, StorageStatuses: storageStatuses, - }, err + }, nil } func shardCheck(shardPath string) (readable bool, writeable bool) { diff --git a/internal/service/server/info_test.go b/internal/service/server/info_test.go index e6aa36f24e..7559d068bc 100644 --- a/internal/service/server/info_test.go +++ b/internal/service/server/info_test.go @@ -1,6 +1,7 @@ package server import ( + "io/ioutil" "net" "testing" @@ -10,6 +11,7 @@ import ( "gitlab.com/gitlab-org/gitaly/internal/config" "gitlab.com/gitlab-org/gitaly/internal/git" "gitlab.com/gitlab-org/gitaly/internal/server/auth" + "gitlab.com/gitlab-org/gitaly/internal/storage" "gitlab.com/gitlab-org/gitaly/internal/testhelper" "gitlab.com/gitlab-org/gitaly/internal/version" "google.golang.org/grpc" @@ -36,6 +38,15 @@ func TestGitalyServerInfo(t *testing.T) { }(config.Config.Storages) config.Config.Storages = testStorages + tempDir, err := ioutil.TempDir("", "gitaly-bin") + require.NoError(t, err) + + config.Config.BinDir = tempDir + + require.NoError(t, storage.WriteMetadataFile(testStorages[0])) + metadata, err := storage.ReadMetadataFile(testStorages[0]) + require.NoError(t, err) + c, err := client.ServerInfo(ctx, &gitalypb.ServerInfoRequest{}) require.NoError(t, err) @@ -52,6 +63,7 @@ func TestGitalyServerInfo(t *testing.T) { require.False(t, c.GetStorageStatuses()[1].Readable) require.False(t, c.GetStorageStatuses()[1].Writeable) + require.Equal(t, metadata.GitalyFilesystemID, c.GetStorageStatuses()[0].FilesystemId) } func runServer(t *testing.T) (*grpc.Server, string) { diff --git a/internal/storage/metadata.go b/internal/storage/metadata.go new file mode 100644 index 0000000000..90abf74f38 --- /dev/null +++ b/internal/storage/metadata.go @@ -0,0 +1,67 @@ +package storage + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + + "github.com/google/uuid" + "gitlab.com/gitlab-org/gitaly/internal/config" + "gitlab.com/gitlab-org/gitaly/internal/safe" +) + +const ( + // metadataFilename is the filename for a file we write on the gitaly server containing metadata about + // the filesystem + metadataFilename = ".gitaly-metadata" +) + +// Metadata contains metadata about the filesystem +type Metadata struct { + GitalyFilesystemID string `json:"gitaly_filesystem_id"` +} + +// WriteMetadataFile marshals and writes a metadata file +func WriteMetadataFile(storage config.Storage) error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + path := filepath.Join(storage.Path, metadataFilename) + + if _, err := os.Stat(path); !os.IsNotExist(err) { + return err + } + + fw, err := safe.CreateFileWriter(ctx, path) + if err != nil { + return err + } + + if err = json.NewEncoder(fw).Encode(&Metadata{ + GitalyFilesystemID: uuid.New().String(), + }); err != nil { + return err + } + + return fw.Commit() +} + +// ReadMetadataFile reads and decodes the json metadata file +func ReadMetadataFile(storage config.Storage) (Metadata, error) { + path := filepath.Join(storage.Path, metadataFilename) + + var metadata Metadata + + metadataFile, err := os.Open(path) + if err != nil { + return metadata, err + } + defer metadataFile.Close() + + if err = json.NewDecoder(metadataFile).Decode(&metadata); err != nil { + return metadata, err + } + + return metadata, nil +} diff --git a/internal/storage/metadata_test.go b/internal/storage/metadata_test.go new file mode 100644 index 0000000000..88bc5f6b38 --- /dev/null +++ b/internal/storage/metadata_test.go @@ -0,0 +1,83 @@ +package storage + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "gitlab.com/gitlab-org/gitaly/internal/config" +) + +func readFilesystemID(t *testing.T, path string) string { + metadata := make(map[string]string) + + f, err := os.Open(filepath.Join(path, metadataFilename)) + require.NoError(t, err) + defer f.Close() + + require.NoError(t, json.NewDecoder(f).Decode(&metadata)) + return metadata["gitaly_filesystem_id"] +} + +func TestWriteMetdataFile(t *testing.T) { + tempDir, err := ioutil.TempDir("", t.Name()) + require.NoError(t, err) + defer func() { + require.NoError(t, os.RemoveAll(tempDir)) + }() + + shard := config.Storage{ + Path: tempDir, + } + + require.NoError(t, WriteMetadataFile(shard)) + require.NotEmpty(t, readFilesystemID(t, tempDir)) +} + +func TestWriteMetadataFile_AlreadyExists(t *testing.T) { + tempDir, err := ioutil.TempDir("", t.Name()) + require.NoError(t, err) + defer func() { + require.NoError(t, os.RemoveAll(tempDir)) + }() + + metadataPath := filepath.Join(tempDir, ".gitaly-metadata") + metadataFile, err := os.Create(metadataPath) + require.NoError(t, err) + + m := Metadata{ + GitalyFilesystemID: uuid.New().String(), + } + + require.NoError(t, json.NewEncoder(metadataFile).Encode(&m)) + require.NoError(t, metadataFile.Close()) + + require.NoError(t, WriteMetadataFile(config.Storage{ + Path: tempDir, + })) + + require.Equal(t, m.GitalyFilesystemID, readFilesystemID(t, tempDir), "WriteMetadataFile should not clobber the existing file") +} + +func TestReadMetadataFile(t *testing.T) { + shard := config.Storage{ + Path: "testdata", + } + + metadata, err := ReadMetadataFile(shard) + require.NoError(t, err) + require.Equal(t, "test filesystem id", metadata.GitalyFilesystemID, "filesystem id should match the harded value in testdata/.gitaly-metadata") +} + +func TestReadMetadataFile_FileNotExists(t *testing.T) { + shard := config.Storage{ + Path: "/path/doesnt/exist", + } + + _, err := ReadMetadataFile(shard) + require.Error(t, err) +} diff --git a/internal/storage/testdata/.gitaly-metadata b/internal/storage/testdata/.gitaly-metadata new file mode 100644 index 0000000000..f11b36eb88 --- /dev/null +++ b/internal/storage/testdata/.gitaly-metadata @@ -0,0 +1 @@ +{"gitaly_filesystem_id":"test filesystem id"} \ No newline at end of file -- GitLab