diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 24d0db5..5f04898 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,7 +29,7 @@ jobs: # TODO Set up Kind for integration tests. steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: actions/setup-go@v5 with: go-version: "1.20" @@ -38,7 +38,7 @@ jobs: run: | os=$(go env GOOS) arch=$(go env GOARCH) - version=1.29.3 + version=1.30.0 curl -L https://storage.googleapis.com/kubebuilder-tools/kubebuilder-tools-${version}-${os}-${arch}.tar.gz | tar -xz -C /tmp/ sudo mv /tmp/kubebuilder /usr/local/kubebuilder - run: make test @@ -50,9 +50,11 @@ jobs: - test - lint if: github.event_name != 'pull_request' + permissions: + packages: write steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Docker Prep id: prep run: | @@ -69,19 +71,19 @@ jobs: if [ "${{ github.event_name }}" = "push" ]; then TAGS="$TAGS,${DOCKER_IMAGE}:sha-${GITHUB_SHA::8}" fi - echo ::set-output name=version::${VERSION} - echo ::set-output name=tags::${TAGS} - echo ::set-output name=created::$(date -u +'%Y-%m-%dT%H:%M:%SZ') + echo version=${VERSION} >> $GITHUB_OUTPUT + echo tags=${TAGS} >> $GITHUB_OUTPUT + echo created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') >> $GITHUB_OUTPUT - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to Github Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v2 with: registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.GHCR_PAT }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v4 with: context: . file: ./Dockerfile diff --git a/components/migrations.go b/components/migrations.go index ec4acf6..a428717 100644 --- a/components/migrations.go +++ b/components/migrations.go @@ -18,18 +18,19 @@ package components import ( "context" + "encoding/json" + "fmt" "strings" "time" cu "github.com/coderanger/controller-utils" "github.com/pkg/errors" - appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" @@ -40,7 +41,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/source" migrationsv1beta1 "github.com/coderanger/migrations-operator/api/v1beta1" - argoprojstubv1alpha1 "github.com/coderanger/migrations-operator/stubs/argoproj/v1alpha1" "github.com/coderanger/migrations-operator/utils" "github.com/coderanger/migrations-operator/webhook" ) @@ -83,6 +83,24 @@ func (comp *migrationsComponent) Setup(ctx *cu.Context, bldr *ctrl.Builder) erro return nil } +func deepCopyJSON(src map[string]interface{}, dest map[string]interface{}) error { + if src == nil { + return errors.New("src is nil. You cannot read from a nil map") + } + if dest == nil { + return errors.New("dest is nil. You cannot insert to a nil map") + } + jsonStr, err := json.Marshal(src) + if err != nil { + return err + } + err = json.Unmarshal(jsonStr, &dest) + if err != nil { + return err + } + return nil +} + func (comp *migrationsComponent) Reconcile(ctx *cu.Context) (cu.Result, error) { obj := ctx.Object.(*migrationsv1beta1.Migrator) @@ -105,16 +123,18 @@ func (comp *migrationsComponent) Reconcile(ctx *cu.Context) (cu.Result, error) { } // Find a template pod to start from. - allPods := &corev1.PodList{} + allPods := &unstructured.UnstructuredList{} + allPods.SetAPIVersion("v1") + allPods.SetKind("Pod") err = ctx.Client.List(ctx, allPods, &client.ListOptions{Namespace: obj.Namespace}) if err != nil { return cu.Result{}, errors.Wrapf(err, "error listing pods in namespace %s", obj.Namespace) } - pods := []*corev1.Pod{} - var templatePod *corev1.Pod + pods := []*unstructured.Unstructured{} + var templatePod *unstructured.Unstructured for i := range allPods.Items { pod := &allPods.Items[i] - labelSet := labels.Set(pod.Labels) + labelSet := labels.Set(pod.GetLabels()) if selector.Matches(labelSet) { pods = append(pods, pod) if templatePod == nil && templateSelector.Matches(labelSet) { @@ -138,17 +158,19 @@ func (comp *migrationsComponent) Reconcile(ctx *cu.Context) (cu.Result, error) { } // Find the template container. - var templateContainer *corev1.Container + templatePodSpecContainers := templatePodSpec["containers"].([]interface{}) + var templateContainer map[string]interface{} if obj.Spec.Container != "" { // Looking for a specific container name. - for _, c := range templatePodSpec.Containers { - if c.Name == obj.Spec.Container { - templateContainer = &c + for _, c := range templatePodSpecContainers { + container := c.(map[string]interface{}) + if container["name"].(string) == obj.Spec.Container { + templateContainer = container break } } - } else if len(templatePodSpec.Containers) > 0 { - templateContainer = &templatePodSpec.Containers[0] + } else if len(templatePodSpecContainers) > 0 { + templateContainer = templatePodSpecContainers[0].(map[string]interface{}) } if templateContainer == nil { // Welp, either nothing matched the name or somehow there are no containers. @@ -156,36 +178,47 @@ func (comp *migrationsComponent) Reconcile(ctx *cu.Context) (cu.Result, error) { } // Build a migration job object. - migrationContainer := templateContainer.DeepCopy() - migrationContainer.Name = "migrations" + migrationContainer := make(map[string]interface{}) + err = deepCopyJSON(templateContainer, migrationContainer) + if err != nil { + return cu.Result{}, errors.Wrap(err, "error copying template container") + } + migrationContainer["name"] = "migrations" if obj.Spec.Image != "" { - migrationContainer.Image = obj.Spec.Image + migrationContainer["image"] = obj.Spec.Image } if obj.Spec.Command != nil { - migrationContainer.Command = *obj.Spec.Command + migrationContainer["command"] = *obj.Spec.Command } if obj.Spec.Args != nil { - migrationContainer.Args = *obj.Spec.Args + migrationContainer["args"] = *obj.Spec.Args } // TODO resources? // Remove the probes since they will rarely work. - migrationContainer.ReadinessProbe = nil - migrationContainer.LivenessProbe = nil - migrationContainer.StartupProbe = nil + migrationContainer["readinessProbe"] = nil + migrationContainer["livenessProbe"] = nil + migrationContainer["startupProbe"] = nil - migrationPodSpec := templatePodSpec.DeepCopy() - migrationPodSpec.Containers = []corev1.Container{*migrationContainer} - migrationPodSpec.RestartPolicy = corev1.RestartPolicyNever + migrationPodSpec := make(map[string]interface{}) + err = deepCopyJSON(templatePodSpec, migrationPodSpec) + if err != nil { + return cu.Result{}, errors.Wrap(err, "error copying template pod spec") + } + migrationPodSpec["containers"] = []map[string]interface{}{migrationContainer} + migrationPodSpec["restartPolicy"] = corev1.RestartPolicyNever // Purge any migration wait initContainers since that would be a yodawg situation. - initContainers := []corev1.Container{} - for _, c := range migrationPodSpec.InitContainers { - if !strings.HasPrefix(c.Name, "migrate-wait-") { - initContainers = append(initContainers, c) + initContainers := []map[string]interface{}{} + if migrationPodSpec["initContainers"] != nil { + for _, c := range migrationPodSpec["initContainers"].([]interface{}) { + container := c.(map[string]interface{}) + if !strings.HasPrefix(container["name"].(string), "migrate-wait-") { + initContainers = append(initContainers, container) + } } } - migrationPodSpec.InitContainers = initContainers + migrationPodSpec["initContainers"] = initContainers // add labels to the job's pod template jobTemplateLabels := map[string]string{"migrations": obj.Name} @@ -205,21 +238,22 @@ func (comp *migrationsComponent) Reconcile(ctx *cu.Context) (cu.Result, error) { } } - migrationJob := &batchv1.Job{ - ObjectMeta: metav1.ObjectMeta{ - Name: obj.Name + "-migrations", - Namespace: obj.Namespace, - Labels: obj.Labels, - Annotations: map[string]string{}, - }, - Spec: batchv1.JobSpec{ - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: jobTemplateLabels, - Annotations: jobTemplateAnnotations, - }, - Spec: *migrationPodSpec, + migrationJobName := obj.Name + "-migrations" + migrationJobNamespace := obj.Namespace + migrationJobImage := migrationContainer["image"].(string) + migrationJob := &unstructured.Unstructured{} + migrationJob.SetAPIVersion("batch/v1") + migrationJob.SetKind("Job") + migrationJob.SetName(migrationJobName) + migrationJob.SetNamespace(migrationJobNamespace) + migrationJob.SetLabels(obj.Labels) + migrationJob.UnstructuredContent()["spec"] = map[string]interface{}{ + "template": map[string]interface{}{ + "metadata": map[string]interface{}{ + "labels": jobTemplateLabels, + "annotations": jobTemplateAnnotations, }, + "spec": migrationPodSpec, }, } err = controllerutil.SetControllerReference(obj, migrationJob, ctx.Scheme) @@ -233,13 +267,13 @@ func (comp *migrationsComponent) Reconcile(ctx *cu.Context) (cu.Result, error) { if err != nil { return cu.Result{}, errors.Wrap(err, "error getting latest migrator for status") } - if uncachedObj.Status.LastSuccessfulMigration == migrationContainer.Image { - ctx.Conditions.SetfTrue(comp.GetReadyCondition(), "MigrationsUpToDate", "Migration %s already run", migrationContainer.Image) + if uncachedObj.Status.LastSuccessfulMigration == migrationJobImage { + ctx.Conditions.SetfTrue(comp.GetReadyCondition(), "MigrationsUpToDate", "Migration %s already run", migrationJobImage) return cu.Result{}, nil } existingJob := &batchv1.Job{} - err = ctx.Client.Get(ctx, types.NamespacedName{Name: migrationJob.Name, Namespace: migrationJob.Namespace}, existingJob) + err = ctx.Client.Get(ctx, types.NamespacedName{Name: migrationJobName, Namespace: migrationJobNamespace}, existingJob) if err != nil { if kerrors.IsNotFound(err) { // Try to start the migrations. @@ -250,11 +284,11 @@ func (comp *migrationsComponent) Reconcile(ctx *cu.Context) (cu.Result, error) { ctx.Conditions.SetfUnknown(comp.GetReadyCondition(), "CreateError", "Error on create, possible conflict: %v", err) return cu.Result{Requeue: true}, nil } - ctx.Events.Eventf(obj, "Normal", "MigrationsStarted", "Started migration job %s/%s using image %s", migrationJob.Namespace, migrationJob.Name, migrationContainer.Image) - ctx.Conditions.SetfFalse(comp.GetReadyCondition(), "MigrationsRunning", "Started migration job %s/%s using image %s", migrationJob.Namespace, migrationJob.Name, migrationContainer.Image) + ctx.Events.Eventf(obj, "Normal", "MigrationsStarted", "Started migration job %s/%s using image %s", migrationJobNamespace, migrationJobName, migrationJobImage) + ctx.Conditions.SetfFalse(comp.GetReadyCondition(), "MigrationsRunning", "Started migration job %s/%s using image %s", migrationJobNamespace, migrationJobName, migrationJobImage) return cu.Result{}, nil } else { - return cu.Result{}, errors.Wrapf(err, "error getting existing migration job %s/%s", migrationJob.Namespace, migrationJob.Name) + return cu.Result{}, errors.Wrapf(err, "error getting existing migration job %s/%s", migrationJobNamespace, migrationJobName) } } @@ -263,15 +297,15 @@ func (comp *migrationsComponent) Reconcile(ctx *cu.Context) (cu.Result, error) { if len(existingJob.Spec.Template.Spec.Containers) > 0 { existingImage = existingJob.Spec.Template.Spec.Containers[0].Image } - if existingImage == "" || existingImage != migrationContainer.Image { + if existingImage == "" || existingImage != migrationJobImage { // Old, stale migration. Remove it and try again. policy := metav1.DeletePropagationForeground err = ctx.Client.Delete(ctx, existingJob, &client.DeleteOptions{PropagationPolicy: &policy}) if err != nil { return cu.Result{}, errors.Wrapf(err, "error deleting stale migration job %s/%s", existingJob.Namespace, existingJob.Name) } - ctx.Events.Eventf(obj, "Normal", "StaleJob", "Deleted stale migration job %s/%s (%s)", migrationJob.Namespace, migrationJob.Name, existingImage) - ctx.Conditions.SetfFalse(comp.GetReadyCondition(), "StaleJob", "Deleted stale migration job %s/%s (%s)", migrationJob.Namespace, migrationJob.Name, existingImage) + ctx.Events.Eventf(obj, "Normal", "StaleJob", "Deleted stale migration job %s/%s (%s)", migrationJobNamespace, migrationJobName, existingImage) + ctx.Conditions.SetfFalse(comp.GetReadyCondition(), "StaleJob", "Deleted stale migration job %s/%s (%s)", migrationJobNamespace, migrationJobName, existingImage) return cu.Result{RequeueAfter: 1 * time.Second, SkipRemaining: true}, nil } @@ -284,7 +318,7 @@ func (comp *migrationsComponent) Reconcile(ctx *cu.Context) (cu.Result, error) { } ctx.Events.Eventf(obj, "Normal", "MigrationsSucceeded", "Migration job %s/%s using image %s succeeded", existingJob.Namespace, existingJob.Name, existingImage) ctx.Conditions.SetfTrue(comp.GetReadyCondition(), "MigrationsSucceeded", "Migration job %s/%s using image %s succeeded", existingJob.Namespace, existingJob.Name, existingImage) - obj.Status.LastSuccessfulMigration = migrationContainer.Image + obj.Status.LastSuccessfulMigration = migrationJobImage return cu.Result{}, nil } @@ -301,9 +335,9 @@ func (comp *migrationsComponent) Reconcile(ctx *cu.Context) (cu.Result, error) { return cu.Result{}, nil } -func (_ *migrationsComponent) findOwners(ctx *cu.Context, obj client.Object) ([]client.Object, error) { +func (_ *migrationsComponent) findOwners(ctx *cu.Context, obj *unstructured.Unstructured) ([]*unstructured.Unstructured, error) { namespace := obj.GetNamespace() - owners := []client.Object{} + owners := []*unstructured.Unstructured{} for { owners = append(owners, obj) ref := metav1.GetControllerOfNoCopy(obj) @@ -311,17 +345,10 @@ func (_ *migrationsComponent) findOwners(ctx *cu.Context, obj client.Object) ([] break } gvk := schema.FromAPIVersionAndKind(ref.APIVersion, ref.Kind) - ownerObj, err := ctx.Scheme.New(gvk) - if err != nil { - // Gracefully handle kinds that we haven't registered. Useful when a Rollout or Deployment is - // owned by someone's in-house operator - if runtime.IsNotRegisteredError(err) { - break - } - return nil, errors.Wrapf(err, "error finding object type for owner reference %v", ref) - } - obj = ownerObj.(client.Object) - err = ctx.Client.Get(ctx, types.NamespacedName{Name: ref.Name, Namespace: namespace}, obj) + obj = &unstructured.Unstructured{} + obj.SetGroupVersionKind(gvk) + obj.SetName(ref.Name) // Is this needed? + err := ctx.Client.Get(ctx, types.NamespacedName{Name: ref.Name, Namespace: namespace}, obj) if err != nil { // Gracefully handle objects we don't have access to if kerrors.IsForbidden(err) { @@ -337,34 +364,44 @@ func (_ *migrationsComponent) findOwners(ctx *cu.Context, obj client.Object) ([] return owners, nil } -func (_ *migrationsComponent) findSpecFor(ctx *cu.Context, obj client.Object) *corev1.PodSpec { - switch v := obj.(type) { - case *corev1.Pod: - return &v.Spec - case *appsv1.Deployment: - return &v.Spec.Template.Spec - case *argoprojstubv1alpha1.Rollout: - if v.Spec.WorkloadRef != nil { - if v.Spec.WorkloadRef.Kind == "Deployment" { - deployment := appsv1.Deployment{} - err := ctx.Client.Get(ctx, client.ObjectKey{Namespace: v.Namespace, Name: v.Spec.WorkloadRef.Name}, &deployment) +func (_ *migrationsComponent) findSpecFor(ctx *cu.Context, obj *unstructured.Unstructured) map[string]interface{} { + gvk := obj.GetObjectKind().GroupVersionKind() + switch fmt.Sprintf("%s/%s", gvk.Group, gvk.Kind) { + case "/Pod": + return obj.UnstructuredContent()["spec"].(map[string]interface{}) + case "apps/Deployment": + spec := obj.UnstructuredContent()["spec"].(map[string]interface{}) + template := spec["template"].(map[string]interface{}) + return template["spec"].(map[string]interface{}) + case "argoproj.io/Rollout": + spec := obj.UnstructuredContent()["spec"].(map[string]interface{}) + if spec["workloadRef"] != nil { + workloadRef := spec["workloadRef"].(map[string]interface{}) + workloadKind := workloadRef["kind"].(string) + if workloadKind == "Deployment" { + deployment := &unstructured.Unstructured{} + deployment.SetAPIVersion(workloadRef["apiVersion"].(string)) + deployment.SetKind(workloadKind) + err := ctx.Client.Get(ctx, types.NamespacedName{Name: workloadRef["name"].(string), Namespace: obj.GetNamespace()}, deployment) if err != nil { return nil } - return &deployment.Spec.Template.Spec + deploymentSpec := deployment.UnstructuredContent()["spec"].(map[string]interface{}) + deploymentTemplate := deploymentSpec["template"].(map[string]interface{}) + return deploymentTemplate["spec"].(map[string]interface{}) } else { // TODO handle other WorkloadRef types return nil } } - return &v.Spec.Template.Spec - // TODO other types. lots of them. + template := spec["template"].(map[string]interface{}) + return template["spec"].(map[string]interface{}) default: return nil } } -func (comp *migrationsComponent) findOwnerSpec(ctx *cu.Context, obj client.Object) (*corev1.PodSpec, error) { +func (comp *migrationsComponent) findOwnerSpec(ctx *cu.Context, obj *unstructured.Unstructured) (map[string]interface{}, error) { owners, err := comp.findOwners(ctx, obj) if err != nil { return nil, err diff --git a/components/migrations_test.go b/components/migrations_test.go index df53c9a..0efa371 100644 --- a/components/migrations_test.go +++ b/components/migrations_test.go @@ -28,6 +28,8 @@ import ( corev1 "k8s.io/api/core/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + // "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + // "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" migrationsv1beta1 "github.com/coderanger/migrations-operator/api/v1beta1" @@ -380,4 +382,44 @@ var _ = Describe("Migrations component", func() { helper.TestClient.GetName("testing-migrations", job) Expect(job.Spec.Template.Spec.Containers[0].Image).To(Equal("myapp:v1")) }) + + // It("doesn't remove restartPolicy from init containers", func() { + // upod := &unstructured.Unstructured{} + // upod.SetGroupVersionKind(schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"}) + // upod.SetName(pod.GetName()) + // upod.SetNamespace(pod.GetNamespace()) + // upod.UnstructuredContent()["spec"] = map[string][]map[string]string{ + // "containers": { + // { + // "name": "main", + // "image": "fake", + // }, + // }, + // "initContainers": { + // { + // "name": "sidecar", + // "image": "fake", + // "restartPolicy": "Always", + // }, + // }, + // } + + // helper.TestClient.Create(upod) + // helper.MustReconcile() + // Expect(helper.Events).To(Receive(Equal("Normal MigrationsStarted Started migration job default/testing-migrations using image fake"))) + // Expect(obj).To(HaveCondition("MigrationsReady").WithReason("MigrationsRunning").WithStatus("False")) + + // ujob := &unstructured.Unstructured{} + // ujob.SetGroupVersionKind(schema.GroupVersionKind{Group: "batch", Version: "v1", Kind: "Job"}) + // helper.TestClient.GetName("testing-migrations", ujob) + // spec := ujob.UnstructuredContent()["spec"].(map[string]interface{}) + // template := spec["template"].(map[string]interface{}) + // tSpec := template["spec"].(map[string]interface{}) + // initContainers := (tSpec["initContainers"].([]interface{})) + // containers := tSpec["containers"].([]interface{}) + // initContainer := initContainers[0].(map[string]interface{}) + // container := containers[0].(map[string]interface{}) + // Expect(initContainer["restartPolicy"]).To(Equal("Always")) + // Expect(container["name"]).To(Equal("migrations")) + // }) }) diff --git a/config/rbac/leader_election_role.yaml b/config/rbac/leader_election_role.yaml index eaa7915..70432f3 100644 --- a/config/rbac/leader_election_role.yaml +++ b/config/rbac/leader_election_role.yaml @@ -4,6 +4,15 @@ kind: Role metadata: name: leader-election-role rules: +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - get + - list + - update - apiGroups: - "" resources: diff --git a/webhook/webhook_test.go b/webhook/webhook_test.go index 015c758..c04e0c1 100644 --- a/webhook/webhook_test.go +++ b/webhook/webhook_test.go @@ -306,7 +306,6 @@ var _ = Describe("InitInjector", func() { c.EventuallyGetName("testing", pod) spec := pod.UnstructuredContent()["spec"].(map[string]interface{}) - println(spec) initContainers := spec["initContainers"].([]interface{}) sidecar := initContainers[0].(map[string]interface{}) Expect(sidecar["restartPolicy"]).To(Equal("Always"))