Skip to content

Managed database name generation #7741

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

Managed database name generation #7741

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b0fc604
iImprove database name generation and sanitize user/database identifiers
grahamplata Aug 6, 2025
84cd3f1
use string builder
grahamplata Aug 6, 2025
e221af5
database and user naming in tests
grahamplata Aug 6, 2025
5b57275
revert
grahamplata Aug 7, 2025
dca1f7b
review
grahamplata Aug 7, 2025
0fbbd53
nits
grahamplata Aug 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
86 changes: 43 additions & 43 deletions admin/provisioner/clickhousestatic/provisioner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,16 +130,9 @@ func (p *Provisioner) Provision(ctx context.Context, r *provisioner.Resource, op
}

// Prepare for creating the schema and user.
id := strings.ReplaceAll(r.ID, "-", "")
id := sanitizeName(r.ID)
user := fmt.Sprintf("rill_%s", id)
dbName := fmt.Sprintf("rill_%s", id)

// Use org and project names to create a more human-readable database name.
orgName := sanitizeName(getAnnotationValue(opts.Annotations, "organization_name"))
projectName := sanitizeName(getAnnotationValue(opts.Annotations, "project_name"))
if orgName != "" && projectName != "" {
dbName = fmt.Sprintf("rill_%s_%s_%s", orgName, projectName, id)
}
dbName := generateDatabaseName(id, opts.Annotations)

password := newPassword()
annotationsJSON, err := json.Marshal(opts.Annotations)
Expand All @@ -148,67 +141,41 @@ func (p *Provisioner) Provision(ctx context.Context, r *provisioner.Resource, op
}

// Idempotently create the schema
_, err = p.ch.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s %s COMMENT ?", dbName, p.onCluster()), string(annotationsJSON))
_, err = p.ch.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s` %s COMMENT ?", dbName, p.onCluster()), string(annotationsJSON))
if err != nil {
return nil, fmt.Errorf("failed to create clickhouse database: %w", err)
}

// Idempotently create the user.
_, err = p.ch.ExecContext(ctx, fmt.Sprintf("CREATE USER IF NOT EXISTS %s %s IDENTIFIED WITH sha256_password BY ? DEFAULT DATABASE %s GRANTEES NONE", user, p.onCluster(), dbName), password)
_, err = p.ch.ExecContext(ctx, fmt.Sprintf("CREATE USER IF NOT EXISTS `%s` %s IDENTIFIED WITH sha256_password BY ? DEFAULT DATABASE `%s` GRANTEES NONE", user, p.onCluster(), dbName), password)
if err != nil {
return nil, fmt.Errorf("failed to create clickhouse user: %w", err)
}

// When creating the user, the password assignment is not idempotent (if there are two concurrent invocations, we don't know which password was used).
// By adding the password separately, we ensure all passwords will work.
// NOTE: Requires ClickHouse 24.9 or later.
_, err = p.ch.ExecContext(ctx, fmt.Sprintf("ALTER USER %s %s ADD IDENTIFIED WITH sha256_password BY ?", user, p.onCluster()), password)
_, err = p.ch.ExecContext(ctx, fmt.Sprintf("ALTER USER `%s` %s ADD IDENTIFIED WITH sha256_password BY ?", user, p.onCluster()), password)
if err != nil {
return nil, fmt.Errorf("failed to add password for clickhouse user: %w", err)
}

// Grant privileges on the database to the user
_, err = p.ch.ExecContext(ctx, fmt.Sprintf(`
GRANT %s
SELECT,
INSERT,
ALTER,
CREATE TABLE,
CREATE DICTIONARY,
CREATE VIEW,
DROP TABLE,
DROP DICTIONARY,
DROP VIEW,
TRUNCATE,
OPTIMIZE,
SHOW DICTIONARIES,
dictGet
ON %s.* TO %s
`, p.onCluster(), dbName, user))
_, err = p.ch.ExecContext(ctx, fmt.Sprintf(`GRANT %s SELECT, INSERT, ALTER, CREATE TABLE, CREATE DICTIONARY, CREATE VIEW, DROP TABLE, DROP DICTIONARY, DROP VIEW, TRUNCATE, OPTIMIZE, SHOW DICTIONARIES, dictGet ON `+"`%s`"+`.* TO `+"`%s`", p.onCluster(), dbName, user))
if err != nil {
return nil, fmt.Errorf("failed to grant privileges to clickhouse user: %w", err)
}

// Grant access to system.parts for reporting disk usage.
// NOTE 1: ClickHouse automatically adds row filters to restrict result to tables the user has access to.
// NOTE 2: We do not need to explicitly grant access to system.tables and system.columns because ClickHouse adds those implicitly.
_, err = p.ch.ExecContext(ctx, fmt.Sprintf("GRANT %s SELECT ON system.parts TO %s", p.onCluster(), user))
_, err = p.ch.ExecContext(ctx, fmt.Sprintf("GRANT %s SELECT ON system.parts TO `%s`", p.onCluster(), user))
if err != nil {
return nil, fmt.Errorf("failed to grant system privileges to clickhouse user: %w", err)
}

// Grant some additional global privileges to the user
_, err = p.ch.ExecContext(ctx, fmt.Sprintf(`
GRANT %s
URL,
REMOTE,
MONGO,
MYSQL,
POSTGRES,
S3,
AZURE
ON *.* TO %s
`, p.onCluster(), user))
_, err = p.ch.ExecContext(ctx, fmt.Sprintf(`GRANT %s URL, REMOTE, MONGO, MYSQL, POSTGRES, S3, AZURE ON *.* TO `+"`%s`", p.onCluster(), user))
if err != nil {
return nil, fmt.Errorf("failed to grant global privileges to clickhouse user: %w", err)
}
Expand Down Expand Up @@ -269,13 +236,13 @@ func (p *Provisioner) Deprovision(ctx context.Context, r *provisioner.Resource)
}

// Drop the database
_, err = p.ch.ExecContext(ctx, fmt.Sprintf("DROP DATABASE IF EXISTS %s %s", opts.Auth.Database, p.onCluster()))
_, err = p.ch.ExecContext(ctx, fmt.Sprintf("DROP DATABASE IF EXISTS `%s` %s", opts.Auth.Database, p.onCluster()))
if err != nil {
return fmt.Errorf("failed to drop clickhouse database: %w", err)
}

// Drop the user
_, err = p.ch.ExecContext(ctx, fmt.Sprintf("DROP USER IF EXISTS %s %s", opts.Auth.Username, p.onCluster()))
_, err = p.ch.ExecContext(ctx, fmt.Sprintf("DROP USER IF EXISTS `%s` %s", opts.Auth.Username, p.onCluster()))
if err != nil {
return fmt.Errorf("failed to drop clickhouse user: %w", err)
}
Expand Down Expand Up @@ -357,3 +324,36 @@ func sanitizeName(name string) string {

return strings.ToLower(result.String())
}

// generateDatabaseName creates a predictable, length-controlled database name
func generateDatabaseName(resourceID string, annotations map[string]string) string {
orgName := sanitizeName(getAnnotationValue(annotations, "organization_name"))
projectName := sanitizeName(getAnnotationValue(annotations, "project_name"))

var builder strings.Builder
builder.WriteString("rill")

// Build database name based on available components for human readability
if orgName != "" {
builder.WriteString("_")
builder.WriteString(orgName)
}
if projectName != "" {
builder.WriteString("_")
builder.WriteString(projectName)
}
builder.WriteString("_")
builder.WriteString(resourceID)

dbName := builder.String()

// Ensure the total length doesn't exceed 63 characters
if len(dbName) > 63 {
dbName = dbName[:63]
}

// Remove any trailing underscores
dbName = strings.TrimRight(dbName, "_")

return dbName
}
62 changes: 58 additions & 4 deletions admin/provisioner/clickhousestatic/provisioner_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,9 +294,9 @@ func TestClickHouseStaticHumanReadableNaming(t *testing.T) {
opts2, err := clickhouse.ParseDSN(cfg.DSN)
require.NoError(t, err)
// Check that the database name follows the expected format
expectedID := strings.ReplaceAll(resourceID, "-", "")
expectedID := sanitizeName(resourceID)
expectedDBName := fmt.Sprintf("rill_acme_corp_my_project_%s", expectedID)
expectedUser := fmt.Sprintf("rill_%s", expectedID) // User name remains UUID format
expectedUser := fmt.Sprintf("rill_%s", expectedID) // User name uses sanitized format

require.Equal(t, expectedDBName, opts2.Auth.Database)
require.Equal(t, expectedUser, opts2.Auth.Username)
Expand Down Expand Up @@ -377,9 +377,9 @@ func TestClickHouseStaticFallbackNaming(t *testing.T) {
opts2, err := clickhouse.ParseDSN(cfg.DSN)
require.NoError(t, err)
// Check that the database name follows the fallback format
expectedID := strings.ReplaceAll(resourceID, "-", "")
expectedID := sanitizeName(resourceID)
expectedDBName := fmt.Sprintf("rill_%s", expectedID)
expectedUser := fmt.Sprintf("rill_%s", expectedID) // User name always uses UUID format
expectedUser := fmt.Sprintf("rill_%s", expectedID) // User name uses sanitized format

require.Equal(t, expectedDBName, opts2.Auth.Database)
require.Equal(t, expectedUser, opts2.Auth.Username)
Expand Down Expand Up @@ -427,3 +427,57 @@ func TestSanitizeName(t *testing.T) {
})
}
}

func TestGenerateDatabaseName(t *testing.T) {
tests := []struct {
name string
id string
annotations map[string]string
expected string
}{
{
name: "with org and project",
id: "77cf2b72_65ab_4bbe_a10e_627bcff4915e",
annotations: map[string]string{"organization_name": "rilldata", "project_name": "dev-project-1"},
expected: "rill_rilldata_dev_project_1_77cf2b72_65ab_4bbe_a10e_627bcff4915",
},
{
name: "with org only",
id: "12345",
annotations: map[string]string{"organization_name": "acme-corp"},
expected: "rill_acme_corp_12345",
},
{
name: "with project only",
id: "12345",
annotations: map[string]string{"project_name": "my-project"},
expected: "rill_my_project_12345",
},
{
name: "no annotations",
id: "12345",
annotations: map[string]string{},
expected: "rill_12345",
},
{
name: "nil annotations",
id: "12345",
annotations: nil,
expected: "rill_12345",
},
{
name: "long name truncated",
id: "very_long_resource_id_that_will_cause_truncation_12345678",
annotations: map[string]string{"organization_name": "very_long_organization_name", "project_name": "very_long_project_name"},
expected: "rill_very_long_organization_name_very_long_project_name_very_lo",
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := generateDatabaseName(tt.id, tt.annotations)
require.Equal(t, tt.expected, result)
require.LessOrEqual(t, len(result), 63, "database name should not exceed 63 characters")
})
}
}