From d98112cb61f94460c4952eacca97fa11c5e9ff96 Mon Sep 17 00:00:00 2001 From: Farnaz Babaeian Date: Fri, 17 Nov 2023 17:43:28 +0100 Subject: [PATCH] Adding Snapshotting Functionality --- .github/workflows/pr-check.yaml | 4 +- .github/workflows/release.yaml | 6 +- Makefile | 2 +- deploy/k8s/controller-deployment.yaml | 22 ++ deploy/k8s/rbac.yaml | 43 +++ examples/k8s/pod.yaml | 2 + .../k8s/snapshot/0-volumestorageclass.yaml | 6 + .../k8s/snapshot/pod-with-snapshot-data.yaml | 18 ++ examples/k8s/snapshot/pvc-from-snapshot.yaml | 15 + examples/k8s/snapshot/volumesnapshot.yaml | 8 + go.mod | 41 +-- go.sum | 108 +++---- pkg/cloud/cloud.go | 34 ++- pkg/cloud/fake/fake.go | 142 ++++++++- pkg/cloud/snapshots.go | 274 ++++++++++++++++++ pkg/cloud/volumes.go | 29 +- pkg/driver/controller.go | 254 +++++++++++++++- pkg/mount/mount.go | 5 +- pkg/util/idlocker.go | 115 +++++++- 19 files changed, 1010 insertions(+), 118 deletions(-) create mode 100644 examples/k8s/snapshot/0-volumestorageclass.yaml create mode 100644 examples/k8s/snapshot/pod-with-snapshot-data.yaml create mode 100644 examples/k8s/snapshot/pvc-from-snapshot.yaml create mode 100644 examples/k8s/snapshot/volumesnapshot.yaml create mode 100644 pkg/cloud/snapshots.go diff --git a/.github/workflows/pr-check.yaml b/.github/workflows/pr-check.yaml index 899f7ee..d0e5ce1 100644 --- a/.github/workflows/pr-check.yaml +++ b/.github/workflows/pr-check.yaml @@ -8,7 +8,7 @@ jobs: name: Lint runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: @@ -25,7 +25,7 @@ jobs: go-version: "^1.15" - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Cache uses: actions/cache@v3 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0755764..605bea7 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -18,7 +18,7 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Cache uses: actions/cache@v3 @@ -32,7 +32,7 @@ jobs: run: make container - name: Log into registry - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{github.actor}} @@ -77,7 +77,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Create manifest run: | diff --git a/Makefile b/Makefile index 6ffb13a..529505b 100644 --- a/Makefile +++ b/Makefile @@ -31,7 +31,7 @@ clean: .PHONY: build-% $(CMDS:%=build-%): build-%: mkdir -p bin - CGO_ENABLED=0 go build -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*" ./cmd/$* + CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags '$(FULL_LDFLAGS)' -o "./bin/$*" ./cmd/$* .PHONY: container-% $(CMDS:%=container-%): container-%: build-% diff --git a/deploy/k8s/controller-deployment.yaml b/deploy/k8s/controller-deployment.yaml index dd17ee5..ca4934b 100644 --- a/deploy/k8s/controller-deployment.yaml +++ b/deploy/k8s/controller-deployment.yaml @@ -1,3 +1,5 @@ +# In this YAML file CloudStack CSI Controller contains the following sidecars: +# external-attacher, external-provisioner, external-snapshotter, liveness-probe apiVersion: apps/v1 kind: Deployment metadata: @@ -105,6 +107,26 @@ spec: - name: socket-dir mountPath: /var/lib/csi/sockets/pluginproxy/ + - name: external-snapshotter + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.3.1 + imagePullPolicy: IfNotPresent + args: + - --v=4 + - --csi-address=$(ADDRESS) + - --timeout=300s + - --leader-election + - --leader-election-lease-duration=120s + - --leader-election-renew-deadline=60s + - --leader-election-retry-period=30s + - --kube-api-qps=100 + - --kube-api-burst=100 + env: + - name: ADDRESS + value: /var/lib/csi/sockets/pluginproxy/csi.sock + volumeMounts: + - mountPath: /var/lib/csi/sockets/pluginproxy/ + name: socket-dir + - name: liveness-probe image: registry.k8s.io/sig-storage/livenessprobe:v2.10.0 args: diff --git a/deploy/k8s/rbac.yaml b/deploy/k8s/rbac.yaml index 6ab55de..13f36a7 100644 --- a/deploy/k8s/rbac.yaml +++ b/deploy/k8s/rbac.yaml @@ -49,3 +49,46 @@ roleRef: kind: ClusterRole name: cloudstack-csi-controller-role apiGroup: rbac.authorization.k8s.io + +--- +# external snapshotter +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshotter-role +rules: + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + # Secret permission is optional. + # Enable it if your driver needs secret. + # For example, `csi.storage.k8s.io/snapshotter-secret-name` is set in VolumeSnapshotClass. + # See https://kubernetes-csi.github.io/docs/secrets-and-credentials.html for more details. + # - apiGroups: [""] + # resources: ["secrets"] + # verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["create", "get", "list", "watch", "update", "delete", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-snapshotter-binding +subjects: + - kind: ServiceAccount + name: cloudstack-csi-controller + namespace: kube-system +roleRef: + kind: ClusterRole + name: csi-snapshotter-role + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/examples/k8s/pod.yaml b/examples/k8s/pod.yaml index afcc288..05ed14f 100644 --- a/examples/k8s/pod.yaml +++ b/examples/k8s/pod.yaml @@ -6,6 +6,8 @@ spec: containers: - name: example image: busybox + command: ["/bin/sh"] + args: ["-c", "while true; do date >> /data/date.txt; sleep 5; done"] volumeMounts: - mountPath: "/data" name: example-volume diff --git a/examples/k8s/snapshot/0-volumestorageclass.yaml b/examples/k8s/snapshot/0-volumestorageclass.yaml new file mode 100644 index 0000000..7ecba23 --- /dev/null +++ b/examples/k8s/snapshot/0-volumestorageclass.yaml @@ -0,0 +1,6 @@ +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshotClass +metadata: + name: csi-cloudstack-snapclass +deletionPolicy: Delete +driver: csi.cloudstack.apache.org \ No newline at end of file diff --git a/examples/k8s/snapshot/pod-with-snapshot-data.yaml b/examples/k8s/snapshot/pod-with-snapshot-data.yaml new file mode 100644 index 0000000..86541eb --- /dev/null +++ b/examples/k8s/snapshot/pod-with-snapshot-data.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: example-pod-from-snapshot-data +spec: + containers: + - name: example + image: busybox + volumeMounts: + - mountPath: "/data" + name: example-volume + stdin: true + stdinOnce: true + tty: true + volumes: + - name: example-volume + persistentVolumeClaim: + claimName: pvc-from-snapshot \ No newline at end of file diff --git a/examples/k8s/snapshot/pvc-from-snapshot.yaml b/examples/k8s/snapshot/pvc-from-snapshot.yaml new file mode 100644 index 0000000..0f7d4d8 --- /dev/null +++ b/examples/k8s/snapshot/pvc-from-snapshot.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-from-snapshot +spec: + accessModes: + - ReadWriteOnce + storageClassName: cloudstack-custom + resources: + requests: + storage: 1Gi + dataSource: + kind: VolumeSnapshot + name: example-snapshot + apiGroup: snapshot.storage.k8s.io \ No newline at end of file diff --git a/examples/k8s/snapshot/volumesnapshot.yaml b/examples/k8s/snapshot/volumesnapshot.yaml new file mode 100644 index 0000000..d1a4108 --- /dev/null +++ b/examples/k8s/snapshot/volumesnapshot.yaml @@ -0,0 +1,8 @@ +apiVersion: snapshot.storage.k8s.io/v1 +kind: VolumeSnapshot +metadata: + name: example-snapshot +spec: + volumeSnapshotClassName: csi-cloudstack-snapclass + source: + persistentVolumeClaimName: example-pvc \ No newline at end of file diff --git a/go.mod b/go.mod index 60be039..1d48922 100644 --- a/go.mod +++ b/go.mod @@ -7,14 +7,16 @@ require ( github.com/container-storage-interface/spec v1.8.0 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/hashicorp/go-uuid v1.0.3 + github.com/kubernetes-csi/csi-lib-utils v0.16.0 github.com/kubernetes-csi/csi-test/v4 v4.4.0 go.uber.org/zap v1.25.0 - golang.org/x/text v0.12.0 - google.golang.org/grpc v1.57.0 + golang.org/x/text v0.14.0 + google.golang.org/grpc v1.59.0 + google.golang.org/protobuf v1.31.0 gopkg.in/gcfg.v1 v1.2.3 - k8s.io/api v0.27.5 - k8s.io/apimachinery v0.27.5 - k8s.io/client-go v0.27.5 + k8s.io/api v0.28.0 + k8s.io/apimachinery v0.28.0 + k8s.io/client-go v0.28.0 k8s.io/mount-utils v0.27.5 k8s.io/utils v0.0.0-20230726121419-3b25d923346b ) @@ -23,17 +25,17 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.9.0 // indirect github.com/fsnotify/fsnotify v1.4.9 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.1 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/gnostic v0.5.7-v3refs // indirect + github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.5.9 // indirect - github.com/google/gofuzz v1.1.0 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -44,24 +46,23 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nxadm/tail v1.4.8 // indirect github.com/onsi/ginkgo v1.16.5 // indirect - github.com/onsi/gomega v1.27.4 // indirect + github.com/onsi/gomega v1.27.6 // indirect github.com/spf13/pflag v1.0.5 // indirect go.uber.org/multierr v1.10.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/oauth2 v0.7.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/term v0.7.0 // indirect - golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect + golang.org/x/net v0.18.0 // indirect + golang.org/x/oauth2 v0.11.0 // indirect + golang.org/x/sys v0.14.0 // indirect + golang.org/x/term v0.14.0 // indirect + golang.org/x/time v0.3.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/klog/v2 v2.90.1 // indirect - k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect + k8s.io/klog/v2 v2.100.1 // indirect + k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect diff --git a/go.sum b/go.sum index 636c10d..62bc454 100644 --- a/go.sum +++ b/go.sum @@ -18,7 +18,6 @@ github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/container-storage-interface/spec v1.6.0 h1:vwN9uCciKygX/a0toYryoYD5+qI9ZFeAMuhEEKO+JBA= github.com/container-storage-interface/spec v1.6.0/go.mod h1:8K96oQNkJ7pFcC2R9Z1ynGGBB1I93kcS6PGg3SsOk8s= github.com/container-storage-interface/spec v1.8.0 h1:D0vhF3PLIZwlwZEf2eNbpujGCNwspwTYf2idJRJx4xI= github.com/container-storage-interface/spec v1.8.0/go.mod h1:ROLik+GhPslwwWRNFF1KasPzroNARibH2rfz1rkg4H0= @@ -26,7 +25,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -42,17 +40,17 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= -github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= @@ -75,8 +73,8 @@ github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaS github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -87,13 +85,16 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= -github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -111,13 +112,14 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kubernetes-csi/csi-lib-utils v0.16.0 h1:LXCvkhXHtFOkl7LoDqFdho/MuebccZqWxLwhKiRGiBg= +github.com/kubernetes-csi/csi-lib-utils v0.16.0/go.mod h1:fp1Oik+45tP2o4X9SD/SBWXLTQYT9wtLxGasBE3+vBI= github.com/kubernetes-csi/csi-test/v4 v4.4.0 h1:r0mnAwDURI24Vw3a/LyA/ga11yD5ZGuU7+REO35Na9s= github.com/kubernetes-csi/csi-test/v4 v4.4.0/go.mod h1:t1RzseMZJKy313nezI/d7TolbbiKpUZM3SXQvXxOX0w= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -140,24 +142,23 @@ github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vv github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= -github.com/onsi/ginkgo/v2 v2.9.1 h1:zie5Ly042PD3bsCvsSOPvRnFwyo3rKe64TJlD6nu0mk= +github.com/onsi/ginkgo/v2 v2.9.4 h1:xR7vG4IXt5RWx6FfIjyAtsoMAtnc3C/rFXBBd2AjZwE= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= -github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= -github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ= +github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -169,8 +170,8 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -211,12 +212,14 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= +golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -245,21 +248,21 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= +golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44= -golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -271,7 +274,7 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -285,10 +288,9 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201209185603-f92720507ed4/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19 h1:0nDDozoAU19Qb2HwhXadU8OcsiO/09cnTqhUtq2MEOM= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230525234030-28d5490b6b19/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17 h1:Jyp0Hsi0bmHXG6k9eATXoYtjd6e2UzZ1SCn/wIupY14= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:oQ5rr10WTTMvP4A36n8JpR1OrO1BEiV4f78CneXZxkA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= @@ -298,8 +300,10 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -313,11 +317,12 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -337,23 +342,22 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.27.5 h1:49hIzqJNSuOQpA53MMihgAS4YDcQitTy58B9PMFthLc= -k8s.io/api v0.27.5/go.mod h1:zjBZB+c0KDU55Wxb9Bob9WZGxu9zdKHitzHxBtaIVoA= -k8s.io/apimachinery v0.27.5 h1:6Q5HBXYJJPisd6yDVAprLe6FQsmw7a7Cu69dcrpQET8= -k8s.io/apimachinery v0.27.5/go.mod h1:XNfZ6xklnMCOGGFNqXG7bUrQCoR04dh/E7FprV6pb+E= -k8s.io/client-go v0.27.5 h1:sH/fkqzk35kuf0GPx+dZuN7fhEswBSAVCrWFq3E1km0= -k8s.io/client-go v0.27.5/go.mod h1:u+IKnqPZSPw51snIMKiIAV8LQQ+hya5bvxpOOPTUXPI= +k8s.io/api v0.28.0 h1:3j3VPWmN9tTDI68NETBWlDiA9qOiGJ7sdKeufehBYsM= +k8s.io/api v0.28.0/go.mod h1:0l8NZJzB0i/etuWnIXcwfIv+xnDOhL3lLW919AWYDuY= +k8s.io/apimachinery v0.28.0 h1:ScHS2AG16UlYWk63r46oU3D5y54T53cVI5mMJwwqFNA= +k8s.io/apimachinery v0.28.0/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= +k8s.io/client-go v0.28.0 h1:ebcPRDZsCjpj62+cMk1eGNX1QkMdRmQ6lmz5BLoFWeM= +k8s.io/client-go v0.28.0/go.mod h1:0Asy9Xt3U98RypWJmU1ZrRAGKhP6NqDPmptlAzK2kMc= k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/klog/v2 v2.90.1 h1:m4bYOKall2MmOiRaR1J+We67Do7vm9KiQVlT96lnHUw= -k8s.io/klog/v2 v2.90.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= -k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= +k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= k8s.io/mount-utils v0.27.5 h1:6g98ViXeqbhQ8gd/IWuW2+ksMBxFOtVEokSkNGtOQ4Y= k8s.io/mount-utils v0.27.5/go.mod h1:vmcjYdi2Vg1VTWY7KkhvwJVY6WDHxb/QQhiQKkR8iNs= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= diff --git a/pkg/cloud/cloud.go b/pkg/cloud/cloud.go index c376303..18c1da3 100644 --- a/pkg/cloud/cloud.go +++ b/pkg/cloud/cloud.go @@ -18,10 +18,17 @@ type Interface interface { GetVolumeByID(ctx context.Context, volumeID string) (*Volume, error) GetVolumeByName(ctx context.Context, name string) (*Volume, error) - CreateVolume(ctx context.Context, diskOfferingID, zoneID, name string, sizeInGB int64) (string, error) + CreateVolume(ctx context.Context, diskOfferingID, zoneID, name string, sizeInGB int64, snapshotID string) (string, error) DeleteVolume(ctx context.Context, id string) error AttachVolume(ctx context.Context, volumeID, vmID string) (string, error) DetachVolume(ctx context.Context, volumeID string) error + + CreateSnapshot(ctx context.Context, name, volumeID string) (string, error) + ListSnapshots(ctx context.Context, filters map[string]string) ([]Snapshot, string, error) + DeleteSnapshot(ctx context.Context, snapshotID string) error + WaitSnapshotReady(ctx context.Context, snapshotID string) error + GetSnapshotByID(ctx context.Context, snapshotID string) (*Snapshot, error) + GetSnapshotByName(ctx context.Context, name string) (*Snapshot, error) } // Volume represents a CloudStack volume. @@ -37,6 +44,7 @@ type Volume struct { VirtualMachineID string DeviceID string + SnapshotID string } // VM represents a CloudStack Virtual Machine. @@ -45,6 +53,30 @@ type VM struct { ZoneID string } +// Snapshot contains all the information associated with a CloudStack Snapshot. +type Snapshot struct { + // Unique identifier. + ID string `json:"id"` + + // Date created. + Created string `json:"created"` + + // Display name. + Name string `json:"name"` + + // ID of the Volume from which this Snapshot was created. + VolumeID string `json:"volume_id"` + + // ID of the Zone. + ZoneID string `json:"zone_id"` + + // Currect state of the Snapshot. + State string `json:"state"` + + // Size of the Snapshot, in GB. + VirtualSize int `json:"size"` +} + // Specific errors var ( ErrNotFound = errors.New("not found") diff --git a/pkg/cloud/fake/fake.go b/pkg/cloud/fake/fake.go index b2fb7b3..cb6b9ce 100644 --- a/pkg/cloud/fake/fake.go +++ b/pkg/cloud/fake/fake.go @@ -4,6 +4,8 @@ package fake import ( "context" + "strconv" + "time" "github.com/hashicorp/go-uuid" @@ -12,11 +14,14 @@ import ( ) const zoneID = "a1887604-237c-4212-a9cd-94620b7880fa" +const snapshotReadyStatus = "BackedUp" type fakeConnector struct { - node *cloud.VM - volumesByID map[string]cloud.Volume - volumesByName map[string]cloud.Volume + node *cloud.VM + volumesByID map[string]cloud.Volume + volumesByName map[string]cloud.Volume + snapshotsByID map[string]cloud.Snapshot + snapshotsByName map[string]cloud.Snapshot } // New returns a new fake implementation of the @@ -31,14 +36,25 @@ func New() cloud.Interface { VirtualMachineID: "", DeviceID: "", } + snapshot := cloud.Snapshot{ + ID: "fc8621ac-168f-4f51-b3cf-c11681609c12", + Created: "2006-01-02T15:04:05-0700", + Name: "snapshot-1", + VolumeID: "CSIVolumeID", + ZoneID: zoneID, + VirtualSize: 10, + State: "Backedup", + } node := &cloud.VM{ ID: "0d7107a3-94d2-44e7-89b8-8930881309a5", ZoneID: zoneID, } return &fakeConnector{ - node: node, - volumesByID: map[string]cloud.Volume{volume.ID: volume}, - volumesByName: map[string]cloud.Volume{volume.Name: volume}, + node: node, + volumesByID: map[string]cloud.Volume{volume.ID: volume}, + volumesByName: map[string]cloud.Volume{volume.Name: volume}, + snapshotsByID: map[string]cloud.Snapshot{snapshot.ID: snapshot}, + snapshotsByName: map[string]cloud.Snapshot{snapshot.Name: snapshot}, } } @@ -73,14 +89,26 @@ func (f *fakeConnector) GetVolumeByName(ctx context.Context, name string) (*clou return nil, cloud.ErrNotFound } -func (f *fakeConnector) CreateVolume(ctx context.Context, diskOfferingID, zoneID, name string, sizeInGB int64) (string, error) { +func (f *fakeConnector) CreateVolume(ctx context.Context, diskOfferingID, zoneID, name string, sizeInGB int64, snapshotID string) (string, error) { id, _ := uuid.GenerateUUID() - vol := cloud.Volume{ - ID: id, - Name: name, - Size: util.GigaBytesToBytes(sizeInGB), - DiskOfferingID: diskOfferingID, - ZoneID: zoneID, + vol := cloud.Volume{} + if snapshotID != "" { + vol = cloud.Volume{ + ID: id, + Name: name, + Size: util.GigaBytesToBytes(sizeInGB), + DiskOfferingID: diskOfferingID, + ZoneID: zoneID, + SnapshotID: snapshotID, + } + } else { + vol = cloud.Volume{ + ID: id, + Name: name, + Size: util.GigaBytesToBytes(sizeInGB), + DiskOfferingID: diskOfferingID, + ZoneID: zoneID, + } } f.volumesByID[vol.ID] = vol f.volumesByName[vol.Name] = vol @@ -103,3 +131,91 @@ func (f *fakeConnector) AttachVolume(ctx context.Context, volumeID, vmID string) func (f *fakeConnector) DetachVolume(ctx context.Context, volumeID string) error { return nil } + +func (f *fakeConnector) CreateSnapshot(ctx context.Context, name, volID string) (string, error) { + id, _ := uuid.GenerateUUID() + createdAt := time.Now().Format("2006-01-02T15:04:05-0700") + + snap := cloud.Snapshot{ + ID: id, + Name: name, + VolumeID: volID, + Created: createdAt, + State: "Backedup", + } + f.snapshotsByID[snap.ID] = snap + f.snapshotsByName[snap.Name] = snap + return snap.ID, nil +} + +func (f *fakeConnector) DeleteSnapshot(ctx context.Context, snapID string) error { + if snap, ok := f.snapshotsByID[snapID]; ok { + name := snap.Name + delete(f.snapshotsByName, name) + } + delete(f.snapshotsByID, snapID) + return nil +} + +func (f *fakeConnector) WaitSnapshotReady(ctx context.Context, snapshotName string) error { + snap, ok := f.snapshotsByName[snapshotName] + if ok && snap.State == snapshotReadyStatus { + return nil + } + return nil +} + +func (f *fakeConnector) GetSnapshotByID(ctx context.Context, snapshotID string) (*cloud.Snapshot, error) { + snap, ok := f.snapshotsByID[snapshotID] + if ok { + return &snap, nil + } + return nil, cloud.ErrNotFound +} + +func (f *fakeConnector) GetSnapshotByName(ctx context.Context, name string) (*cloud.Snapshot, error) { + snap, ok := f.snapshotsByName[name] + if ok { + return &snap, nil + } + return nil, cloud.ErrNotFound +} + +func (f *fakeConnector) ListSnapshots(ctx context.Context, filters map[string]string) ([]cloud.Snapshot, string, error) { + var snaplist []cloud.Snapshot + + name := filters["Name"] + volumeID := filters["VolumeID"] + startingToken := filters["Marker"] + limitfilter := filters["Limit"] + limit, _ := strconv.Atoi(limitfilter) + + for _, value := range f.snapshotsByID { + if volumeID != "" { + if value.VolumeID == volumeID { + snaplist = append(snaplist, value) + break + } + } else if name != "" { + if value.Name == name { + snaplist = append(snaplist, value) + break + } + } else { + snaplist = append(snaplist, value) + } + } + + if startingToken != "" && len(snaplist) > limit { + t, _ := strconv.Atoi(startingToken) + snaplist = snaplist[t:] + } + + var nextPageToken string + + if limit != 0 { + snaplist = snaplist[:limit] + nextPageToken = limitfilter + } + return snaplist, nextPageToken, nil +} diff --git a/pkg/cloud/snapshots.go b/pkg/cloud/snapshots.go new file mode 100644 index 0000000..7e2357d --- /dev/null +++ b/pkg/cloud/snapshots.go @@ -0,0 +1,274 @@ +package cloud + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/apache/cloudstack-go/v2/cloudstack" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "k8s.io/apimachinery/pkg/util/wait" +) + +const ( + snapshotReadyStatus = "BackedUp" + snapReadyDuration = 1 * time.Second + snapReadyFactor = 1.2 + snapReadySteps = 10 + asyncBackup = true +) + +// CreateSnapshot issues a request to take a Snapshot of the specified Volume with the corresponding ID and +// returns the resultant CloudStack Snapshot Item upon success +func (c *client) CreateSnapshot(ctx context.Context, name, volumeID string) (string, error) { + // Input validation + if name == "" || volumeID == "" { + return "", status.Errorf(codes.Internal, "Snapshotname %s; requested for volume %s", name, volumeID) + } + + // Pre-API Call Checks + // Retrieve volume information to ensure it exists and is in a valid state + volume, _, err := c.Volume.GetVolumeByID(volumeID) + if err != nil { + return "", fmt.Errorf("failed to retrieve volume '%s': %v", volumeID, err) + } + + // Check if the volume is in a 'Ready' state for snapshot creation + if volume.State != "Ready" { + return "", fmt.Errorf("volume '%s' is not in a 'Ready' state for snapshot creation", volumeID) + } + + // Preparing snapshot creation parameters + p := c.Snapshot.NewCreateSnapshotParams(volumeID) + p.SetName(name) + p.SetVolumeid(volumeID) + p.SetAsyncbackup(asyncBackup) + ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "CreateSnapshot", "params", map[string]string{ + "name": name, + "volumeid": volumeID, + "asyncbackup": strconv.FormatBool(asyncBackup), + }) + snapshot, err := c.Snapshot.CreateSnapshot(p) + if err != nil { + return "", fmt.Errorf("failed to create snapshot '%s' for volume '%s': %v", name, volumeID, err) + } + + return snapshot.Id, nil +} + +// ListSnapshots retrieves a list of active snapshots from CloudStack for the corresponding Domain. We also +// provide the ability to provide limit to enable the consumer to provide accurate pagination. +// In addition the filters argument provides a mechanism for passing in valid filter strings to the list +// operation. Valid filter keys are: Name, VolumeID, Limit, Marker (DomainID has no effect) + +func (c *client) ListSnapshots(ctx context.Context, filters map[string]string) ([]Snapshot, string, error) { + var snapshotList []Snapshot + var nextPageToken string + + // Initialize parameters for CloudStack API call + p := c.Snapshot.NewListSnapshotsParams() + + // Declare pointer variables for pageSize and currentPage + pageSize := 20 // Default pageSize + currentPage := 1 // Default to the first page + var err error + + // Apply filters and pagination parameters + for key, value := range filters { + switch key { + case "Name": + p.SetName(value) + case "VolumeID": + p.SetVolumeid(value) + case "Marker": + // Try to parse the page number from the filters + currentPage, err = strconv.Atoi(value) + if err != nil { + fmt.Printf("Invalid format for Marker: %s, using default page\n", value) + currentPage = 1 // Reassign to default if parsing fails + } + p.SetPage(currentPage) + case "Limit": + // Try to parse the page size from the filters + pageSize, err = strconv.Atoi(value) + if err != nil { + fmt.Printf("Invalid or unsupported format for Limit: %s, using default limit\n", value) + pageSize = 20 // Reassign to default if parsing fails or unsupported value + } + p.SetPagesize(pageSize) + default: + fmt.Printf("Not a valid filter key %s\n", key) + } + } + + // Log the final pagination settings + fmt.Printf("Fetching snapshots with Page: %d, PageSize: %d\n", currentPage, pageSize) + + // Call CloudStack API to list snapshots + resp, err := c.Snapshot.ListSnapshots(p) + if err != nil { + return nil, "", fmt.Errorf("error listing snapshots from CloudStack: %w", err) + } + + // Convert the response to your []Snapshot type + for _, apiSnapshot := range resp.Snapshots { + snapshot := convertAPISnapshotToSnapshot(apiSnapshot) + snapshotList = append(snapshotList, snapshot) + } + + // Determine the nextPageToken + if morePagesExist(resp, pageSize, currentPage) { + // Set nextPageToken to the next page + nextPageToken = strconv.Itoa(currentPage + 1) + if nextPageToken != "" { + fmt.Println("The nextPageToken is set, more than one page is available.") + + } + + } + + return snapshotList, nextPageToken, nil +} + +// DeleteSnapshot issues a request to delete the Snapshot with the specified ID from the backend +func (c *client) DeleteSnapshot(ctx context.Context, snapshotID string) error { + p := c.Snapshot.NewDeleteSnapshotParams(snapshotID) + ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "DeleteSnapshot", "params", map[string]string{ + "id": snapshotID, + }) + _, err := c.Snapshot.DeleteSnapshot(p) + if err != nil && strings.Contains(err.Error(), "4350") { + // CloudStack error InvalidParameterValueException + return ErrNotFound + } + return err +} + +// GetSnapshotByID returns snapshot details by id +func (c *client) GetSnapshotByID(ctx context.Context, snapshotID string) (*Snapshot, error) { + p := c.Snapshot.NewListSnapshotsParams() + p.SetId(snapshotID) + ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "ListSnapshots", "params", map[string]string{ + "id": snapshotID, + }) + l, err := c.Snapshot.ListSnapshots(p) + if err != nil { + return nil, err + } + if l.Count == 0 { + return nil, ErrNotFound + } + if l.Count > 1 { + return nil, ErrTooManyResults + } + snap := l.Snapshots[0] + + v := Snapshot{ + ID: snap.Id, + Name: snap.Name, + VirtualSize: int(snap.Virtualsize), + Created: snap.Created, + ZoneID: snap.Zoneid, + VolumeID: snap.Volumeid, + State: snap.State, + } + return &v, nil +} + +// GetSnapshotByName returns snapshot details by name +func (c *client) GetSnapshotByName(ctx context.Context, name string) (*Snapshot, error) { + p := c.Snapshot.NewListSnapshotsParams() + p.SetName(name) + ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "ListSnapshots", "params", map[string]string{ + "name": name, + }) + l, err := c.Snapshot.ListSnapshots(p) + if err != nil { + return nil, err + } + if l.Count == 0 { + return nil, ErrNotFound + } + if l.Count > 1 { + return nil, ErrTooManyResults + } + snap := l.Snapshots[0] + + v := Snapshot{ + ID: snap.Id, + Name: snap.Name, + VirtualSize: int(snap.Virtualsize), + Created: snap.Created, + ZoneID: snap.Zoneid, + VolumeID: snap.Volumeid, + State: snap.State, + } + return &v, nil +} + +// WaitSnapshotReady waits till snapshot is ready +func (c *client) WaitSnapshotReady(ctx context.Context, snapshotID string) error { + backoff := wait.Backoff{ + Duration: snapReadyDuration, + Factor: snapReadyFactor, + Steps: snapReadySteps, + } + + err := wait.ExponentialBackoff(backoff, func() (bool, error) { + ready, err := c.snapshotIsReady(ctx, snapshotID) + if err != nil { + return false, err + } + return ready, nil + }) + + if wait.Interrupted(err) { + err = fmt.Errorf("timeout, snapshot %s is still not Ready %v", snapshotID, err.Error()) + } + + return err +} + +func (c *client) snapshotIsReady(ctx context.Context, snapshotID string) (bool, error) { + snap, err := c.GetSnapshotByID(ctx, snapshotID) + if err != nil { + return false, fmt.Errorf("Snapshot is not ready: %w", err) + } + + return snap.State == snapshotReadyStatus, nil +} + +// convertAPISnapshotToSnapshot converts an API snapshot object to your application's Snapshot type. +func convertAPISnapshotToSnapshot(apiSnapshot *cloudstack.Snapshot) Snapshot { + // Conversion logic here + return Snapshot{ + ID: apiSnapshot.Id, + Created: apiSnapshot.Created, + Name: apiSnapshot.Name, + VolumeID: apiSnapshot.Volumeid, + ZoneID: apiSnapshot.Zoneid, + State: apiSnapshot.State, + VirtualSize: int(apiSnapshot.Virtualsize), + } +} + +// morePagesExist determines if there are more pages of results based on the API response. +func morePagesExist(resp *cloudstack.ListSnapshotsResponse, pageSize int, currentPage int) bool { + if resp == nil || pageSize <= 0 { + return false + } + + totalSnapshots := resp.Count + totalPages := totalSnapshots / pageSize + // Check for any remaining snapshots not fitting in a full page + if totalSnapshots%pageSize != 0 { + totalPages++ + } + fmt.Printf("Total Page is %d", totalPages) + + return currentPage < totalPages +} diff --git a/pkg/cloud/volumes.go b/pkg/cloud/volumes.go index 394be39..ee114a5 100644 --- a/pkg/cloud/volumes.go +++ b/pkg/cloud/volumes.go @@ -66,18 +66,31 @@ func (c *client) GetVolumeByName(ctx context.Context, name string) (*Volume, err return &v, nil } -func (c *client) CreateVolume(ctx context.Context, diskOfferingID, zoneID, name string, sizeInGB int64) (string, error) { +func (c *client) CreateVolume(ctx context.Context, diskOfferingID, zoneID, name string, sizeInGB int64, snapshotID string) (string, error) { p := c.Volume.NewCreateVolumeParams() - p.SetDiskofferingid(diskOfferingID) p.SetZoneid(zoneID) p.SetName(name) p.SetSize(sizeInGB) - ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "CreateVolume", "params", map[string]string{ - "diskofferingid": diskOfferingID, - "zoneid": zoneID, - "name": name, - "size": strconv.FormatInt(sizeInGB, 10), - }) + if snapshotID != "" { + // No need to set Disk Offering ID for creating volume from snapshot. + // Disk Offering ID cannot be passed whilst creating volume from snapshot other than ROOT disk snapshots. + p.SetSnapshotid(snapshotID) + ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "CreateVolume", "params", map[string]string{ + "diskofferingid": diskOfferingID, + "zoneid": zoneID, + "name": name, + "size": strconv.FormatInt(sizeInGB, 10), + "snapshotid": snapshotID, + }) + } else { + p.SetDiskofferingid(diskOfferingID) + ctxzap.Extract(ctx).Sugar().Infow("CloudStack API call", "command", "CreateVolume", "params", map[string]string{ + "diskofferingid": diskOfferingID, + "zoneid": zoneID, + "name": name, + "size": strconv.FormatInt(sizeInGB, 10), + }) + } vol, err := c.Volume.CreateVolume(p) if err != nil { return "", err diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index 5386ad6..84dc05a 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -4,11 +4,15 @@ import ( "context" "fmt" "math/rand" + "strconv" + "time" "github.com/container-storage-interface/spec/lib/go/csi" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "github.com/kubernetes-csi/csi-lib-utils/protosanitizer" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" "github.com/leaseweb/cloudstack-csi-driver/pkg/cloud" "github.com/leaseweb/cloudstack-csi-driver/pkg/util" @@ -84,7 +88,7 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol VolumeId: vol.ID, CapacityBytes: vol.Size, VolumeContext: req.GetParameters(), - // ContentSource: req.GetVolumeContentSource(), TODO: snapshot support + ContentSource: req.GetVolumeContentSource(), AccessibleTopology: []*csi.Topology{ Topology{ZoneID: vol.ZoneID}.ToCSI(), }, @@ -125,25 +129,46 @@ func (cs *controllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol } zoneID = t.ZoneID } + content := req.GetVolumeContentSource() + var snapshotID string - ctxzap.Extract(ctx).Sugar().Infow("Creating new volume", - "name", name, - "size", sizeInGB, - "offering", diskOfferingID, - "zone", zoneID, - ) + if content != nil && content.GetSnapshot() != nil { + snapshotID = content.GetSnapshot().GetSnapshotId() + _, err := cs.connector.GetSnapshotByID(ctx, snapshotID) + if err != nil { + if err == cloud.ErrNotFound { + return nil, status.Errorf(codes.NotFound, "VolumeContentSource Snapshot %s not found", snapshotID) + } + return nil, status.Errorf(codes.Internal, "Failed to retrieve the snapshot %s: %v", snapshotID, err) + } else { + ctxzap.Extract(ctx).Sugar().Infow("Creating new volume from snapshot", + "name", name, + "size", sizeInGB, + "offering", diskOfferingID, + "zone", zoneID, + "snapshotid", snapshotID, + ) + } + } else { + ctxzap.Extract(ctx).Sugar().Infow("Creating new volume", + "name", name, + "size", sizeInGB, + "offering", diskOfferingID, + "zone", zoneID, + ) + } - volID, err := cs.connector.CreateVolume(ctx, diskOfferingID, zoneID, name, sizeInGB) + volumeID, err := cs.connector.CreateVolume(ctx, diskOfferingID, zoneID, name, sizeInGB, snapshotID) if err != nil { return nil, status.Errorf(codes.Internal, "Cannot create volume %s: %v", name, err.Error()) } return &csi.CreateVolumeResponse{ Volume: &csi.Volume{ - VolumeId: volID, + VolumeId: volumeID, CapacityBytes: util.GigaBytesToBytes(sizeInGB), VolumeContext: req.GetParameters(), - // ContentSource: req.GetVolumeContentSource(), TODO: snapshot support + ContentSource: req.GetVolumeContentSource(), AccessibleTopology: []*csi.Topology{ Topology{ZoneID: zoneID}.ToCSI(), }, @@ -430,6 +455,215 @@ func (cs *controllerServer) ControllerGetCapabilities(ctx context.Context, req * }, }, }, + { + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT, + }, + }, + }, + { + Type: &csi.ControllerServiceCapability_Rpc{ + Rpc: &csi.ControllerServiceCapability_RPC{ + Type: csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS, + }, + }, + }, }, }, nil } + +func (cs *controllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error) { + ctxzap.Extract(ctx).Sugar().Infow("CreateSnapshot: called with args", "args", protosanitizer.StripSecrets(*req)) + name := req.Name + volumeID := req.GetSourceVolumeId() + snapshotCreateLock := util.NewOperationLock(ctx) + + if name == "" { + return nil, status.Error(codes.InvalidArgument, "Snapshot name must be provided in CreateSnapshot request") + } + + if volumeID == "" { + return nil, status.Error(codes.InvalidArgument, "VolumeID must be provided in CreateSnapshot request") + } + + err := snapshotCreateLock.GetSnapshotCreateLock(volumeID) + if err != nil { + + ctxzap.Extract(ctx).Sugar().Errorf(util.VolumeOperationAlreadyExistsFmt, volumeID) + return nil, status.Errorf(codes.Aborted, util.VolumeOperationAlreadyExistsFmt, volumeID) + } + defer snapshotCreateLock.ReleaseSnapshotCreateLock(volumeID) + + // Check if a snapshot with that name already exists + if snap, err := cs.connector.GetSnapshotByName(ctx, name); err == cloud.ErrNotFound { + // The snapshot does not exist + } else if err != nil { + // Error with CloudStack + return nil, status.Errorf(codes.Internal, "CloudStack error: %v", err) + } else { + // The snapshot exists. Check if it suits the request. + if snap.VolumeID != volumeID { + // Snapshot exists with a different source volume ID + return nil, status.Errorf(codes.AlreadyExists, "Snapshot with given name already exists, with different source volume ID") + } + s, err := toCSISnapshot(snap) + if err != nil { + return nil, status.Errorf(codes.Internal, + "couldn't convert CloudStack snapshot to CSI snapshot: %s", err.Error()) + } + // Existing snapshot is ok + return &csi.CreateSnapshotResponse{ + Snapshot: s, + }, nil + } + + // Verify a snapshot with the provided name doesn't already exist for this domain + var snap *cloud.Snapshot + filters := map[string]string{} + filters["Name"] = name + snapshots, _, err := cs.connector.ListSnapshots(ctx, filters) + if err != nil { + return nil, status.Error(codes.Internal, "Failed to get snapshots") + } + + if len(snapshots) == 1 { + snap = &snapshots[0] + + if snap.VolumeID != volumeID { + return nil, status.Error(codes.AlreadyExists, "Snapshot with given name already exists, with different source volume ID") + } + } else if len(snapshots) > 1 { + return nil, status.Errorf(codes.Internal, "Multiple snapshots reported by CSI with same name(%s)", name) + + } else { + snapID, err := cs.connector.CreateSnapshot(ctx, name, volumeID) + if err != nil { + return nil, status.Errorf(codes.Internal, "Cannot create volume %s: %v", name, err.Error()) + } + snap, err = cs.connector.GetSnapshotByID(ctx, snapID) + if err != nil { + return nil, status.Errorf(codes.Internal, "Cannot find snapshot %s: %v", snap.Name, err.Error()) + } + + fmt.Printf("CreateSnapshot %s with ID %s from volume with ID: %s", name, snapID, volumeID) + } + + s, err := toCSISnapshot(snap) + if err != nil { + return nil, status.Errorf(codes.Internal, + "couldn't convert CloudStack snapshot to CSI snapshot: %s", err.Error()) + } + err = cs.connector.WaitSnapshotReady(ctx, snap.ID) + if err != nil { + return nil, status.Error(codes.Internal, fmt.Sprintf("CreateSnapshot failed with error %v", err)) + } + + return &csi.CreateSnapshotResponse{ + Snapshot: s, + }, nil +} + +func (cs *controllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error) { + ctxzap.Extract(ctx).Sugar().Infow("DeleteSnapshot called with args", "args", protosanitizer.StripSecrets(*req)) + + snapshotDeleteLock := util.NewOperationLock(ctx) + snapshotID := req.GetSnapshotId() + if snapshotID == "" { + return nil, status.Error(codes.InvalidArgument, "DeleteSnapshot Snapshot ID must be provided") + } + + errLock := snapshotDeleteLock.GetSnapshotDeleteLock(snapshotID) + if errLock != nil { + ctxzap.Extract(ctx).Sugar().Errorf(util.SnapshotOperationAlreadyExistsFmt, snapshotID) + return nil, status.Errorf(codes.Aborted, util.SnapshotOperationAlreadyExistsFmt, snapshotID) + } + defer snapshotDeleteLock.ReleaseSnapshotDeleteLock(snapshotID) + + err := cs.connector.DeleteSnapshot(ctx, snapshotID) + if err != nil && err != cloud.ErrNotFound { + return nil, status.Errorf(codes.Internal, "Cannot delete volume %s: %s", snapshotID, err.Error()) + } + return &csi.DeleteSnapshotResponse{}, nil +} + +func (cs *controllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error) { + snapshotID := req.GetSnapshotId() + if len(snapshotID) != 0 { + snap, err := cs.connector.GetSnapshotByID(ctx, snapshotID) + if err != nil { + if err == cloud.ErrNotFound { + fmt.Printf("Snapshot %s not found", snapshotID) + return &csi.ListSnapshotsResponse{}, nil + } + return nil, status.Errorf(codes.Internal, "Failed to GetSnapshot %s: %v", snapshotID, err) + } + + s, err := toCSISnapshot(snap) + if err != nil { + return nil, status.Errorf(codes.Internal, "Couldn't convert CloudStack snapshot to CSI snapshot: %s", err.Error()) + } + entry := &csi.ListSnapshotsResponse_Entry{ + Snapshot: s, + } + + entries := []*csi.ListSnapshotsResponse_Entry{entry} + return &csi.ListSnapshotsResponse{ + Entries: entries, + }, nil + + } + + filters := map[string]string{} + if volID := req.GetSourceVolumeId(); len(volID) != 0 { + filters["VolumeID"] = volID + } + + var maxEntries int32 + if req.MaxEntries > 0 { + maxEntries = req.MaxEntries + } + + // Marker is the last-seen item and Limit is a page size + filters["Limit"] = strconv.Itoa(int(maxEntries)) + filters["Marker"] = req.StartingToken + + // Fetch all snapshots from CloudStack + allSnapshotsList, nextPageToken, err := cs.connector.ListSnapshots(ctx, filters) + if err != nil { + return nil, status.Errorf(codes.Internal, "error listing snapshots: %v", err) + } + var sentries []*csi.ListSnapshotsResponse_Entry + for _, v := range allSnapshotsList { + s, err := toCSISnapshot(&v) + if err != nil { + return nil, status.Errorf(codes.Internal, "couldn't convert CloudStack snapshot to CSI snapshot: %s", err.Error()) + } + sentry := &csi.ListSnapshotsResponse_Entry{ + Snapshot: s, + } + sentries = append(sentries, sentry) + } + return &csi.ListSnapshotsResponse{ + Entries: sentries, + NextToken: nextPageToken, + }, nil +} + +// toCSISnapshot converts a CloudStack Snapshot struct into a csi.Snapshot struct +func toCSISnapshot(snap *cloud.Snapshot) (*csi.Snapshot, error) { + createdAt, err := time.Parse("2006-01-02T15:04:05-0700", snap.Created) + if err != nil { + return nil, fmt.Errorf("couldn't parse snapshot's created field: %s", err.Error()) + } + + tstamp := timestamppb.New(createdAt) + return &csi.Snapshot{ + SizeBytes: int64(snap.VirtualSize), + SnapshotId: snap.ID, + SourceVolumeId: snap.VolumeID, + CreationTime: tstamp, + ReadyToUse: true, + GroupSnapshotId: "", + }, nil +} diff --git a/pkg/mount/mount.go b/pkg/mount/mount.go index 6a18540..1b143a7 100644 --- a/pkg/mount/mount.go +++ b/pkg/mount/mount.go @@ -5,7 +5,6 @@ package mount import ( "context" "fmt" - "io/ioutil" "os" "path/filepath" "strings" @@ -107,11 +106,11 @@ func (m *mounter) probeVolume(ctx context.Context) { log.Debug("Scaning SCSI host...") scsiPath := "/sys/class/scsi_host/" - if dirs, err := ioutil.ReadDir(scsiPath); err == nil { + if dirs, err := os.ReadDir(scsiPath); err == nil { for _, f := range dirs { name := scsiPath + f.Name() + "/scan" data := []byte("- - -") - if err = ioutil.WriteFile(name, data, 0666); err != nil { + if err = os.WriteFile(name, data, 0666); err != nil { log.Warnf("Failed to rescan scsi host %s", name) } } diff --git a/pkg/util/idlocker.go b/pkg/util/idlocker.go index 802b50b..3711630 100644 --- a/pkg/util/idlocker.go +++ b/pkg/util/idlocker.go @@ -64,14 +64,50 @@ func (vl *VolumeLocks) Release(volumeID string) { vl.locks.Delete(volumeID) } +// SnapshotLocks implements a map with atomic operations. It stores a set of all snapshot IDs +// with an ongoing operation. +type SnapshotLocks struct { + locks sets.Set[string] + mux sync.Mutex +} + +// NewSnapshotLocks returns new SnapshotLocks. +func NewSnapshotLocks() *SnapshotLocks { + return &SnapshotLocks{ + locks: sets.New[string](), + } +} + +// TryAcquire tries to acquire the lock for operating on SnapshotLocks and returns true if successful. +// If another operation is already using SnapshotLocks, returns false. +func (sl *SnapshotLocks) TryAcquire(snapshotID string) bool { + sl.mux.Lock() + defer sl.mux.Unlock() + if sl.locks.Has(snapshotID) { + return false + } + sl.locks.Insert(snapshotID) + + return true +} + +// Release deletes the lock on SnapshotLocks. +func (sl *SnapshotLocks) Release(snapshotID string) { + sl.mux.Lock() + defer sl.mux.Unlock() + sl.locks.Delete(snapshotID) +} + type operation string const ( - createOp operation = "create" - deleteOp operation = "delete" - cloneOpt operation = "clone" - restoreOp operation = "restore" - expandOp operation = "expand" + createOp operation = "create" + deleteOp operation = "delete" + cloneOpt operation = "clone" + restoreOp operation = "restore" + expandOp operation = "expand" + templateCreationOp operation = "templatecreation" + volumeCreationOp operation = "volumecreation" ) // OperationLock implements a map with atomic operations. @@ -178,6 +214,47 @@ func (ol *OperationLock) tryAcquire(op operation, volumeID string) error { return nil } +// snapshotTryAcquire tries to acquire the lock for operating on snapshotID and returns true if successful. +// If another operation is already using snapshotID, returns false. +func (ol *OperationLock) snapshotTryAcquire(op operation, snapshotID string) error { + ol.mux.Lock() + defer ol.mux.Unlock() + switch op { + case deleteOp: + // During delete operation the snapshot should not be under template creation or volume creation. + // Check any template creation operation is going on for given snapshotID + if _, ok := ol.locks[templateCreationOp][snapshotID]; ok { + return fmt.Errorf("an Create Template operation with given id %s already exists", snapshotID) + } + // Check any volume creation operation is going on for given snapshotID + if _, ok := ol.locks[volumeCreationOp][snapshotID]; ok { + return fmt.Errorf("a Create Volume operation with given id %s already exists", snapshotID) + } + ol.locks[deleteOp][snapshotID] = 1 + case templateCreationOp: + // During Template creation operation, controller make sure no volume snapshot deletion happens on the + // referred snapshot, so we are safe from source snapshot delete. + // check any delete operation is going on for given snapshot ID + if _, ok := ol.locks[deleteOp][snapshotID]; ok { + return fmt.Errorf("a Delete operation with given id %s already exists", snapshotID) + } + // increment the counter for Templare Creation operation + val := ol.locks[templateCreationOp][snapshotID] + ol.locks[cloneOpt][snapshotID] = val + 1 + case volumeCreationOp: + // During Volume Creation operation the snapshot should not be deleted + // check any delete operation is going on for given snapshot ID + if _, ok := ol.locks[deleteOp][snapshotID]; ok { + return fmt.Errorf("a Delete operation with given id %s already exists", snapshotID) + } + ol.locks[volumeCreationOp][snapshotID] = 1 + default: + return fmt.Errorf("%v operation not supported", op) + } + + return nil +} + // GetSnapshotCreateLock gets the snapshot lock on given volumeID. func (ol *OperationLock) GetSnapshotCreateLock(volumeID string) error { return ol.tryAcquire(createOp, volumeID) @@ -206,6 +283,11 @@ func (ol *OperationLock) GetExpandLock(volumeID string) error { return ol.tryAcquire(expandOp, volumeID) } +// GetSnapshotDeleteLock gets the snapshot lock on given snapshotID. +func (ol *OperationLock) GetSnapshotDeleteLock(snapshotID string) error { + return ol.snapshotTryAcquire(deleteOp, snapshotID) +} + // ReleaseSnapshotCreateLock releases the create lock on given volumeID. func (ol *OperationLock) ReleaseSnapshotCreateLock(volumeID string) { ol.release(createOp, volumeID) @@ -231,6 +313,11 @@ func (ol *OperationLock) ReleaseExpandLock(volumeID string) { ol.release(expandOp, volumeID) } +// ReleaseDeleteLock releases the delete lock on given snapshotID. +func (ol *OperationLock) ReleaseSnapshotDeleteLock(snapshotID string) { + ol.releaseSnapshot(deleteOp, snapshotID) +} + // release deletes the lock on volumeID. func (ol *OperationLock) release(op operation, volumeID string) { ol.mux.Lock() @@ -248,3 +335,21 @@ func (ol *OperationLock) release(op operation, volumeID string) { ctxzap.Extract(ol.ctx).Sugar().Errorf("%v operation not supported", op) } } + +// releaseSnapshot deletes the lock on snapshotID. +func (ol *OperationLock) releaseSnapshot(op operation, snapshotID string) { + ol.mux.Lock() + defer ol.mux.Unlock() + switch op { + case templateCreationOp, volumeCreationOp, deleteOp: + if val, ok := ol.locks[op][snapshotID]; ok { + // decrement the counter for operation + ol.locks[op][snapshotID] = val - 1 + if ol.locks[op][snapshotID] == 0 { + delete(ol.locks[op], snapshotID) + } + } + default: + ctxzap.Extract(ol.ctx).Sugar().Errorf("%v operation not supported", op) + } +}