diff --git a/docs/README.md b/docs/README.md index 67d65ae18e9f..5e7ba5586c36 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,7 +22,7 @@ The site will be available at http://localhost:3000/docs/master/. ## Build ```bash -# Validate the REST Catalog OpenAPI contract +# Validate the REST OpenAPI contracts yarn test:rest-openapi # Production build diff --git a/docs/docs/concepts/rest/index.md b/docs/docs/concepts/rest/index.md index bd8ada4b6712..8ed168cd4434 100644 --- a/docs/docs/concepts/rest/index.md +++ b/docs/docs/concepts/rest/index.md @@ -63,4 +63,5 @@ RESTCatalog supports multiple access authentication methods, including the follo ## REST Open API -See [REST API](./rest-api). +See [REST Catalog API](./rest-api) for catalog operations and +[REST Management API](./management-api) for permission management. diff --git a/docs/docs/concepts/rest/management-api.md b/docs/docs/concepts/rest/management-api.md new file mode 100644 index 000000000000..2bc71f5d01ce --- /dev/null +++ b/docs/docs/concepts/rest/management-api.md @@ -0,0 +1,559 @@ +--- +title: "REST Management API" +hide_table_of_contents: true +--- + + + +The REST Management API is an experimental OpenAPI 3.1 control-plane extension for object +privileges, row filters, and column masks in a Paimon REST Catalog. Its current contract version is +`1.0` and may evolve incompatibly while the design is being validated. + +`RESTCatalog` exposes `permissionManagement()` and `policyManagement()` directly. These methods are +intentionally not part of the generic `Catalog` interface. A non-REST catalog therefore reports an +unsupported-operation error when a management procedure is called. + +## Catalog addressing + +All management endpoints use the opaque `prefix` returned by the REST Catalog config endpoint. It +is not a catalog name in a payload and is independent of the local engine alias such as `paimon` in +`CALL paimon.sys...`. + +``` +GET /v1/{prefix}/permissions +POST /v1/{prefix}/permissions/grant +POST /v1/{prefix}/permissions/revoke + +GET /v1/{prefix}/databases/{database}/tables/{table}/policies +POST /v1/{prefix}/databases/{database}/tables/{table}/policies +DELETE /v1/{prefix}/databases/{database}/tables/{table}/policies +``` + +Policies are currently attached only to tables. The path is the attachment identity, so policy +request bodies do not repeat a catalog, database, table, or resource type. Catalog- and +database-level matching can be added later with explicit matching semantics instead of implied +path inheritance. + +The complete wire contract is available in +[`rest-management-open-api.yaml`](/rest-management-open-api.yaml). + +## Privileges and policies are independent + +A permission grants one access on one resource to one principal. A data policy restricts rows or +columns visible through an already-authorized read. Creating a policy never grants `SELECT`, and +revoking `SELECT` does not delete policies. + +This separation also defines the expected query path: + +1. The server evaluates object privileges. +2. The server resolves all row-filter and column-masking policies applicable to the caller. +3. The existing REST Catalog table authorization endpoint returns the stored Paimon predicate and + column transforms to the engine. +4. The engine applies those restrictions when planning the scan. + +Management payloads use the same serialized Paimon `Predicate` and `Transform` representation as +the existing `AuthTableQueryResponse`. Policy conflict detection, schema validation, and principal +resolution are server responsibilities. + +## Permission model + +Permission resources are structured objects: + +| Resource type | Required locator | Example | +| --- | --- | --- | +| `CATALOG` | none | `{"type":"CATALOG"}` | +| `CATALOG_ALL` | none | `{"type":"CATALOG_ALL"}` | +| `DATABASE` | `database` | `{"type":"DATABASE","database":"sales"}` | +| `DATABASE_ALL` | `database` | `{"type":"DATABASE_ALL","database":"sales"}` | +| `TABLE` | `database`, `table` | `{"type":"TABLE","database":"sales","table":"orders"}` | +| `COLUMN` | `database`, `table` | `{"type":"COLUMN","database":"sales","table":"orders"}` | +| `FUNCTION` | `database`, `function` | `{"type":"FUNCTION","database":"sales","function":"calculate_tax"}` | +| `VIEW` | `database`, `view` | `{"type":"VIEW","database":"sales","view":"daily_orders"}` | + +Principals are opaque, canonical strings that are globally unique in the server namespace. Their +format is server-defined and may encode a user, group, role, or service identity, for example +`role:analyst` or an external identity-provider ARN. Principal type and membership resolution are +server responsibilities. Access values are limited to 32 characters and principals to 128 +characters. An implementation may resolve wire locators and principals to different stable +persistence identifiers; those internal ids are not exposed by this API. + +The built-in accesses use a common data-authorization vocabulary. Creation accesses intentionally +use their persisted names without underscores: + +| Access | Meaning | +| --- | --- | +| `ALL` | All accesses applicable to the resource. | +| `CREATEDATABASE` | Create a database in a catalog. | +| `DESCRIBE` | Read database metadata or select the current database. | +| `ALTER` | Modify resource metadata. | +| `DROP` | Drop the resource. | +| `CREATETABLE` | Create a table in a database. | +| `CREATEFUNCTION` | Create a function in a database. | +| `CREATEVIEW` | Create a view in a database. | +| `LIST` | List resources in a database. | +| `SELECT` | Read table or view data, or use a function. | +| `UPDATE` | Write table data, including insert, update, and delete operations. | +| `GRANT` | Grant or revoke assignments on the resource. | + +Java and Spark helpers accept access names case-insensitively and normalize them before sending. +The REST wire format uses upper case. Built-in accesses are resource-specific: + +| Resource | Accesses | +| --- | --- | +| `CATALOG` | `ALL`, `ALTER`, `DROP`, `GRANT`, `CREATEDATABASE` | +| `CATALOG_ALL` | `ALL`, `DESCRIBE`, `ALTER`, `DROP`, `GRANT`, `CREATETABLE`, `CREATEVIEW`, `CREATEFUNCTION`, `LIST`, `SELECT`, `UPDATE` | +| `DATABASE` | `ALL`, `DESCRIBE`, `ALTER`, `DROP`, `GRANT`, `CREATETABLE`, `CREATEVIEW`, `CREATEFUNCTION`, `LIST` | +| `DATABASE_ALL` | `ALL`, `SELECT`, `UPDATE`, `ALTER`, `DROP`, `GRANT` | +| `TABLE` | `ALL`, `SELECT`, `UPDATE`, `ALTER`, `DROP`, `GRANT` | +| `COLUMN` | `SELECT` | +| `VIEW` | `ALL`, `SELECT`, `ALTER`, `DROP`, `GRANT` | +| `FUNCTION` | `ALL`, `SELECT`, `ALTER`, `DROP`, `GRANT` | + +An assignment identity is `resource`, `access`, and `principal`. Granting the same identity replaces +its expiry, and revocation is idempotent. `CATALOG`, `DATABASE`, `TABLE`, `COLUMN`, `VIEW`, and +`FUNCTION` apply only to the exact referenced resource. `CATALOG_ALL` is an explicit scope over the +configured catalog's database, table, view, and function descendants; `DATABASE_ALL` is an explicit +scope over the named database's table, view, and function descendants. These scope assignments also +apply to descendants created later. They remain direct assignments in listing responses; the server +does not synthesize inherited assignments. Resolving group membership and role inheritance remains a +server responsibility. + +### Column permissions + +A column permission uses a `COLUMN` resource whose locator is the containing table, `SELECT` +access, and one `columns` object. Exactly one non-empty list is allowed: + +- `columnNames` is an allowlist. Only the named top-level columns are readable. +- `excludedColumnNames` is a denylist. Every current top-level column except the named columns is + readable. + +For example, this assignment allows only `order_id` and `region`: + +```json +{ + "resource": { + "type": "COLUMN", + "database": "sales", + "table": "orders" + }, + "access": "SELECT", + "principal": "role:analyst", + "columns": { + "columnNames": ["order_id", "region"] + } +} +``` + +The assignment identity remains `(resource, access, principal)`; `columns` is not part of the +identity. Granting the same identity replaces the entire previous allowlist or denylist rather than +merging individual names. Revocation therefore omits `columns` and removes the whole column +assignment. + +All named columns must exist when granted, and the table must enforce query authorization. A server +may enable `query-auth.enabled` atomically with the grant; otherwise it must reject the grant. Column +names refer only to top-level fields. For every effective caller principal, applicable column ranges +are intersected. If any applicable range rejects a selected column, the query fails rather than +silently dropping that column. + +Schema evolution keeps the assignment attached to the stable table identity. Renaming a referenced +column updates its stored name. Dropping a referenced column removes it from the range; if that +would leave the stored list empty, the assignment is removed. An allowlist denies columns added +later, while a denylist allows them, so allowlists are safer when new columns may contain sensitive +data. + +`expireTime`, when present, is an exclusive upper bound evaluated against the REST server clock. +At `now >= expireTime`, the assignment must not authorize access. Expired direct assignments may +remain visible in listings until server cleanup. Timestamps must not be more precise than +milliseconds. + +Resource objects in this API are wire locators, not persistence identities. Servers must bind direct +assignments to a stable internal resource identity: renaming a database, table, function, or view +retains its assignments and subsequent responses use the new locator; dropping it removes its direct +assignments; recreating the same locator does not restore them. + +## Data policy model + +A data policy is attached directly to one table and one principal. It applies whenever that +principal is effective for the caller after the server resolves group and role membership. A +principal can have at most one row filter on a table and at most one column mask on each table +column. A row-filter identity is `(table, ROW_FILTER, principal)`; a column-mask identity is +`(table, COLUMN_MASKING, principal, onColumn)`. + +Each policy contains exactly one typed definition: + +| Definition | Required fields | Result | +| --- | --- | --- | +| `rowFilter` | `predicate` | One serialized Paimon `Predicate`, applied to every scan. | +| `columnMask` | `onColumn`, `transform` | One serialized Paimon `Transform` whose result replaces the protected column. | + +The common field is one `principal`. `rowFilter.predicate` maps directly to one entry in +`AuthTableQueryResponse.filter`. `columnMask.onColumn` and `columnMask.transform` map directly to +one key and value in `AuthTableQueryResponse.columnMasking`. Each JSON value is limited to 60 KiB +in UTF-8. This is Paimon's versioned serialization format rather than SQL text or a portable policy +DSL; clients and servers must use compatible Paimon versions. + +Policy creation must be rejected unless all of these conditions hold: + +1. The target database and table exist. +2. The table has `query-auth.enabled=true`; otherwise a stored policy could be silently bypassed. +3. The referenced principal exists. +4. The predicate or transform is recognized by the server, deserializes to a non-null Paimon + object, and is canonicalized before storage. +5. Every referenced field and `onColumn` exists in the target table, and a transform's output type + matches its protected column. + +These invariants continue to apply for the whole table lifecycle. Servers must bind policies to a +stable table identity, preserve that binding across table renames, and remove the policies when the +table is dropped. A table with policies must reject changes that disable `query-auth.enabled` or +remove or rename a protected or referenced column, unless the policy update and schema change are +performed atomically. If an implementation persists all masks for one principal in one document, +creating or dropping one column mask must atomically preserve masks for other columns. + +At authorization time, all applicable row filters must be combined with logical `AND`. More than +one applicable column mask targeting the same column must fail closed. An invalid, unsupported, or +schema-incompatible predicate or transform must also fail closed rather than omit a restriction. + +This experimental contract deliberately does not define governed tags, catalog/database policy +inheritance, or tag-driven matching. Those features need explicit match conditions and conflict +rules before being added. + +## Spark SQL procedures + +The following examples assume a Spark catalog named `paimon`. Replace it with the catalog name in +`spark.sql.catalog.`. + +### Grant permissions + +`grant_permission` returns one row with `result = true` when the server accepts the assignment. + +Grant permission to create databases in the catalog: + +```sql +CALL paimon.sys.grant_permission( + resource_type => 'CATALOG', + access => 'CREATEDATABASE', + principal => 'role:catalog_user' +); +``` + +Grant read access to every applicable object currently or subsequently created in the catalog. This +does not grant catalog-level operations such as `CREATEDATABASE`: + +```sql +CALL paimon.sys.grant_permission( + resource_type => 'CATALOG_ALL', + access => 'SELECT', + principal => 'role:catalog_reader' +); +``` + +Grant permission to create views in a database with an optional expiration time: + +```sql +CALL paimon.sys.grant_permission( + resource_type => 'DATABASE', + database => 'sales', + access => 'CREATEVIEW', + principal => 'role:data_engineer', + expire_time => '2027-01-01T00:00:00Z' +); +``` + +Grant write access to every applicable table currently or subsequently created in one database. +`DATABASE_ALL` requires `database` but does not accept a table, function, or view locator: + +```sql +CALL paimon.sys.grant_permission( + resource_type => 'DATABASE_ALL', + database => 'sales', + access => 'UPDATE', + principal => 'role:sales_writer' +); +``` + +Grant table, function, and view access with the matching locator: + +```sql +CALL paimon.sys.grant_permission( + resource_type => 'TABLE', + database => 'sales', + table => 'orders', + access => 'SELECT', + principal => 'user:alice' +); + +CALL paimon.sys.grant_permission( + resource_type => 'FUNCTION', + database => 'sales', + function => 'calculate_tax', + access => 'SELECT', + principal => 'role:analyst' +); + +CALL paimon.sys.grant_permission( + resource_type => 'VIEW', + database => 'sales', + view => 'daily_orders', + access => 'SELECT', + principal => 'service:reporting_job' +); +``` + +Grant access to selected columns. This requires table query authorization; named arguments are +recommended because the two column range modes are mutually exclusive: + +```sql +ALTER TABLE paimon.sales.orders +SET TBLPROPERTIES ('query-auth.enabled' = 'true'); + +CALL paimon.sys.grant_permission( + resource_type => 'COLUMN', + database => 'sales', + table => 'orders', + access => 'SELECT', + principal => 'role:analyst', + column_names => array('order_id', 'region') +); +``` + +Use `excluded_column_names` for a denylist. Repeating the grant replaces the preceding allowlist in +one operation: + +```sql +CALL paimon.sys.grant_permission( + resource_type => 'COLUMN', + database => 'sales', + table => 'orders', + access => 'SELECT', + principal => 'role:analyst', + excluded_column_names => array('email', 'phone_number') +); +``` + +### List permissions + +`list_permissions` always addresses one exact resource or explicit descendant scope. Omit optional +filters to list every direct assignment on it; effective assignments inherited from a scope are not +synthesized: + +```sql +CALL paimon.sys.list_permissions( + resource_type => 'TABLE', + database => 'sales', + table => 'orders' +); +``` + +Filter by principal or access: + +```sql +CALL paimon.sys.list_permissions( + resource_type => 'TABLE', + database => 'sales', + table => 'orders', + principal => 'role:analyst', + access => 'SELECT' +); +``` + +List the column range attached to a principal. The result exposes `column_names` and +`excluded_column_names` as `ARRAY` columns, with exactly one populated for a `COLUMN` +assignment: + +```sql +CALL paimon.sys.list_permissions( + resource_type => 'COLUMN', + database => 'sales', + table => 'orders', + principal => 'role:analyst', + access => 'SELECT' +); +``` + +The `next_page_token` output is opaque; pass it back unchanged with the same filters: + +```sql +CALL paimon.sys.list_permissions( + resource_type => 'TABLE', + database => 'sales', + table => 'orders', + max_results => 50, + page_token => 'opaque-token-from-previous-row' +); +``` + +### Revoke permissions + +Supply the same three identity fields used by the grant. `expire_time` is not part of identity. + +```sql +CALL paimon.sys.revoke_permission( + resource_type => 'TABLE', + database => 'sales', + table => 'orders', + access => 'SELECT', + principal => 'role:sales_reader' +); +``` + +Repeating the same call succeeds even when the assignment is already absent. + +Column revocation uses the containing table identity and removes the complete range: + +```sql +CALL paimon.sys.revoke_permission( + resource_type => 'COLUMN', + database => 'sales', + table => 'orders', + access => 'SELECT', + principal => 'role:analyst' +); +``` + +### Create row-filter policies + +Before attaching any policy, enable table query authorization: + +```sql +ALTER TABLE paimon.sales.orders +SET TBLPROPERTIES ('query-auth.enabled' = 'true'); +``` + +`create_policy` accepts the canonical `principal` and a serialized Paimon `Predicate`. The JSON +below is the same representation accepted in one `AuthTableQueryResponse.filter` entry. Named +arguments are recommended because row-filter and column-mask definitions use different fields: + +```sql +CALL paimon.sys.create_policy( + database => 'sales', + table => 'orders', + policy_type => 'ROW_FILTER', + principal => 'group:analysts', + predicate_json => '{"kind":"LEAF","transform":{"name":"FIELD_REF","fieldRef":{"index":1,"name":"region","type":"STRING"}},"function":"EQUAL","literals":["APAC"]}' +); +``` + +The call fails if that principal already has a row filter on the table. Drop the existing policy +before creating a different definition for the same identity. Create another policy for a second +principal with a separate call. + +### Create column-masking policies + +For column masking, `on_column` identifies the protected column and `transform_json` is the same +serialized Paimon `Transform` representation used as an +`AuthTableQueryResponse.columnMasking` value. This example replaces every visible phone number +with a fixed string: + +```sql +CALL paimon.sys.create_policy( + database => 'sales', + table => 'customers', + policy_type => 'COLUMN_MASKING', + principal => 'role:support', + on_column => 'phone_number', + transform_json => '{"name":"CONCAT","inputs":["****"]}' +); +``` + +A transform may reference table fields by name. The server remaps their indices to the current +schema, rejects missing fields, and verifies that the result type matches `on_column`: + +```sql +CALL paimon.sys.create_policy( + database => 'sales', + table => 'customers', + policy_type => 'COLUMN_MASKING', + principal => 'group:support', + on_column => 'email', + transform_json => '{"name":"CONCAT","inputs":[{"index":1,"name":"region","type":"STRING"},"-masked"]}' +); +``` + +`predicate_json` is required only for `ROW_FILTER`. `on_column` and `transform_json` are required +only for `COLUMN_MASKING`. JSON containing a single quote must escape it as `''` inside the SQL +string literal. + +### List policies + +List every policy directly attached to one table: + +```sql +CALL paimon.sys.list_policies( + database => 'sales', + table => 'orders' +); +``` + +Filter by policy type or principal. A `column` filter is valid only with +`policy_type => 'COLUMN_MASKING'`: + +```sql +CALL paimon.sys.list_policies( + database => 'sales', + table => 'orders', + policy_type => 'ROW_FILTER', + principal => 'group:analysts' +); +``` + +The output columns are `database`, `table`, `policy_type`, `principal`, `predicate_json`, +`on_column`, `transform_json`, and `next_page_token`. A row filter has only `predicate_json`; a +column mask has only `on_column` and `transform_json`. Pass an opaque continuation token back +unchanged with the same filters: + +```sql +CALL paimon.sys.list_policies( + database => 'sales', + table => 'orders', + max_results => 50, + page_token => 'opaque-token-from-previous-row' +); +``` + +Management listing follows the existing Paimon pagination contract: an empty page terminates +pagination and therefore has no continuation token. Each Spark procedure returns exactly the page +selected by `page_token`; pass a non-null `next_page_token` back unchanged to retrieve the next page. + +### Drop policies + +Drop an existing policy: + +```sql +CALL paimon.sys.drop_policy( + database => 'sales', + table => 'orders', + policy_type => 'ROW_FILTER', + principal => 'group:analysts' +); +``` + +By default an absent policy is an error. Set `if_exists => true` for an idempotent operation: + +```sql +CALL paimon.sys.drop_policy( + database => 'sales', + table => 'orders', + policy_type => 'ROW_FILTER', + principal => 'group:analysts', + if_exists => true +); +``` + +Creating, dropping, or inspecting permissions and policies requires the server to +authorize the caller for `GRANT` on the relevant resource. Authentication, principal +membership, policy persistence, schema validation, and audit logging remain REST server concerns. diff --git a/docs/scripts/validate-rest-openapi.js b/docs/scripts/validate-rest-openapi.js index 62e3598cf555..a2eb075e63ef 100644 --- a/docs/scripts/validate-rest-openapi.js +++ b/docs/scripts/validate-rest-openapi.js @@ -20,8 +20,7 @@ const fs = require('fs'); const path = require('path'); const yaml = require('js-yaml'); -const specPath = path.resolve(__dirname, '..', 'static', 'rest-catalog-open-api.yaml'); -const spec = yaml.load(fs.readFileSync(specPath, 'utf8')); +const HTTP_METHODS = new Set(['get', 'post', 'put', 'delete', 'patch', 'head', 'options', 'trace']); function check(condition, message) { if (!condition) { @@ -33,82 +32,181 @@ function decodePointerSegment(segment) { return segment.replace(/~1/g, '/').replace(/~0/g, '~'); } -function resolveLocalRef(ref) { - check(ref.startsWith('#/'), `Only local OpenAPI references are supported, found: ${ref}`); - return ref - .slice(2) - .split('/') - .map(decodePointerSegment) - .reduce((current, segment) => { - check( - current && Object.prototype.hasOwnProperty.call(current, segment), - `Unresolved OpenAPI reference: ${ref}`, - ); - return current[segment]; - }, spec); -} +function validateCommon(fileName) { + const specPath = path.resolve(__dirname, '..', 'static', fileName); + const spec = yaml.load(fs.readFileSync(specPath, 'utf8')); + + function checkSpec(condition, message) { + check(condition, `${fileName}: ${message}`); + } -function visit(value) { - if (Array.isArray(value)) { - value.forEach(visit); - return; + function resolveLocalRef(ref) { + checkSpec(ref.startsWith('#/'), `Only local OpenAPI references are supported, found: ${ref}`); + return ref + .slice(2) + .split('/') + .map(decodePointerSegment) + .reduce((current, segment) => { + checkSpec( + current && Object.prototype.hasOwnProperty.call(current, segment), + `Unresolved OpenAPI reference: ${ref}`, + ); + return current[segment]; + }, spec); } - if (!value || typeof value !== 'object') { - return; + + function visit(value) { + if (Array.isArray(value)) { + value.forEach(visit); + return; + } + if (!value || typeof value !== 'object') { + return; + } + checkSpec( + !Object.prototype.hasOwnProperty.call(value, 'nullable'), + 'OpenAPI 3.1 schemas must not use nullable', + ); + if (typeof value.$ref === 'string') { + resolveLocalRef(value.$ref); + } + Object.values(value).forEach(visit); } - check(!Object.prototype.hasOwnProperty.call(value, 'nullable'), 'OpenAPI 3.1 schemas must not use nullable'); - if (typeof value.$ref === 'string') { - resolveLocalRef(value.$ref); + + function schema(name) { + const value = spec.components && spec.components.schemas && spec.components.schemas[name]; + checkSpec(value, `Missing OpenAPI schema: ${name}`); + return value; } - Object.values(value).forEach(visit); -} -function schema(name) { - const value = spec.components && spec.components.schemas && spec.components.schemas[name]; - check(value, `Missing OpenAPI schema: ${name}`); - return value; -} + function requireProperties(schemaName, names) { + const properties = schema(schemaName).properties || {}; + names.forEach((name) => { + checkSpec(properties[name], `Schema ${schemaName} is missing property: ${name}`); + }); + return properties; + } -function requireProperties(schemaName, names) { - const properties = schema(schemaName).properties || {}; - names.forEach((name) => { - check(properties[name], `Schema ${schemaName} is missing property: ${name}`); - }); - return properties; -} + function requireRequiredProperties(schemaName, names) { + const required = schema(schemaName).required || []; + names.forEach((name) => { + checkSpec(required.includes(name), `Schema ${schemaName} must require property: ${name}`); + }); + } -function requireTypedIntegerProperties(schemaName, names) { - const properties = requireProperties(schemaName, names); - names.forEach((name) => { - check( - properties[name].type === 'integer' && properties[name].format === 'int64', - `Schema ${schemaName}.${name} must be an int64 integer`, + function requireTypedIntegerProperties(schemaName, names) { + const properties = requireProperties(schemaName, names); + names.forEach((name) => { + checkSpec( + properties[name].type === 'integer' && properties[name].format === 'int64', + `Schema ${schemaName}.${name} must be an int64 integer`, + ); + }); + } + + function requireSchemaReference(schemaName, composition, referencedSchemaName) { + const references = schema(schemaName)[composition] || []; + const expected = `#/components/schemas/${referencedSchemaName}`; + checkSpec( + references.some((reference) => reference.$ref === expected), + `Schema ${schemaName}.${composition} is missing reference: ${expected}`, ); - }); -} + } + + function validatePathParameters(resourcePath, pathItem, operation) { + const templateNames = Array.from(resourcePath.matchAll(/\{([^}]+)\}/g), (match) => match[1]); + const parameters = [...(pathItem.parameters || []), ...(operation.parameters || [])].map( + (parameter) => (parameter.$ref ? resolveLocalRef(parameter.$ref) : parameter), + ); + const pathParameters = parameters.filter((parameter) => parameter.in === 'path'); + templateNames.forEach((name) => { + const parameter = pathParameters.find((candidate) => candidate.name === name); + checkSpec(parameter, `Path ${resourcePath} is missing path parameter: ${name}`); + checkSpec( + parameter.required === true, + `Path parameter ${resourcePath}.${name} must be required`, + ); + }); + pathParameters.forEach((parameter) => { + checkSpec( + templateNames.includes(parameter.name), + `Path ${resourcePath} declares unused path parameter: ${parameter.name}`, + ); + }); + } -function requireSchemaReference(schemaName, composition, referencedSchemaName) { - const references = schema(schemaName)[composition] || []; - const expected = `#/components/schemas/${referencedSchemaName}`; - check( - references.some((reference) => reference.$ref === expected), - `Schema ${schemaName}.${composition} is missing reference: ${expected}`, + checkSpec(spec.openapi === '3.1.1', `Expected OpenAPI 3.1.1, found: ${spec.openapi}`); + checkSpec( + spec.paths && spec.components && spec.components.schemas, + 'Incomplete OpenAPI document', ); + visit(spec); + + const operations = new Map(); + for (const [resourcePath, pathItem] of Object.entries(spec.paths)) { + for (const [method, operation] of Object.entries(pathItem)) { + if (!HTTP_METHODS.has(method)) { + continue; + } + validatePathParameters(resourcePath, pathItem, operation); + checkSpec( + operation.operationId, + `Operation ${method.toUpperCase()} ${resourcePath} has no operationId`, + ); + checkSpec( + !operations.has(operation.operationId), + `Duplicate operationId: ${operation.operationId}`, + ); + operations.set(operation.operationId, operation); + } + } + + function requireOperation(operationId) { + const operation = operations.get(operationId); + checkSpec(operation, `Missing operationId: ${operationId}`); + return operation; + } + + function requireResponses(operationId, statusCodes) { + const responses = requireOperation(operationId).responses || {}; + statusCodes.forEach((statusCode) => { + checkSpec( + responses[statusCode], + `Operation ${operationId} is missing response: ${statusCode}`, + ); + }); + } + + return { + spec, + operations, + schema, + requireOperation, + requireProperties, + requireRequiredProperties, + requireTypedIntegerProperties, + requireSchemaReference, + requireResponses, + checkSpec, + }; } -function requireArrayOfIdentifiers(schemaName, propertyName) { - const properties = requireProperties(schemaName, [propertyName]); - check(properties[propertyName].type === 'array', `Schema ${schemaName}.${propertyName} must be an array`); - check( +function requireArrayOfIdentifiers(contract, schemaName, propertyName) { + const properties = contract.requireProperties(schemaName, [propertyName]); + contract.checkSpec( + properties[propertyName].type === 'array', + `Schema ${schemaName}.${propertyName} must be an array`, + ); + contract.checkSpec( properties[propertyName].items && properties[propertyName].items.$ref === '#/components/schemas/Identifier', `Schema ${schemaName}.${propertyName} items must reference Identifier`, ); } -function requireNullableStringProperty(schemaName, propertyName) { - const property = requireProperties(schemaName, [propertyName])[propertyName]; - check( +function requireNullableStringProperty(contract, schemaName, propertyName) { + const property = contract.requireProperties(schemaName, [propertyName])[propertyName]; + contract.checkSpec( Array.isArray(property.type) && property.type.includes('string') && property.type.includes('null'), @@ -116,85 +214,572 @@ function requireNullableStringProperty(schemaName, propertyName) { ); } -check(spec.openapi === '3.1.1', `Expected OpenAPI 3.1.1, found: ${spec.openapi}`); -check(spec.paths && spec.components && spec.components.schemas, 'Incomplete OpenAPI document'); -visit(spec); +function requireExactEnum(contract, schemaName, expectedValues) { + const actualValues = contract.schema(schemaName).enum || []; + contract.checkSpec( + actualValues.length === expectedValues.length && + expectedValues.every((value) => actualValues.includes(value)), + `Schema ${schemaName} must define enum values: ${expectedValues.join(', ')}`, + ); +} + +function validateCatalogOpenApi() { + const contract = validateCommon('rest-catalog-open-api.yaml'); + [ + 'getConfig', + 'createDatabase', + 'getDatabase', + 'alterDatabase', + 'dropDatabase', + 'createTable', + 'getTable', + 'alterTable', + 'dropTable', + ].forEach(contract.requireOperation); -const operationIds = new Set(); -for (const pathItem of Object.values(spec.paths)) { - for (const operation of Object.values(pathItem)) { - if (!operation || typeof operation !== 'object' || !operation.operationId) { - continue; - } - check(!operationIds.has(operation.operationId), `Duplicate operationId: ${operation.operationId}`); - operationIds.add(operation.operationId); - } + contract.requireProperties('ConfigResponse', ['defaults', 'overrides']); + contract.requireProperties('CreateDatabaseRequest', ['name', 'options']); + contract.requireProperties('AlterDatabaseRequest', ['removals', 'updates']); + contract.requireProperties('CreateTableRequest', ['identifier', 'schema']); + contract.requireProperties('AlterTableRequest', ['changes']); + contract.requireProperties('Identifier', ['database', 'object']); + contract.requireProperties('Schema', [ + 'fields', + 'partitionKeys', + 'primaryKeys', + 'options', + 'comment', + ]); + contract.requireProperties('DataField', ['id', 'name', 'type', 'description', 'defaultValue']); + + contract.requireSchemaReference('DataType', 'oneOf', 'VectorType'); + contract.requireProperties('VectorType', ['type', 'element', 'length']); + contract.requireSchemaReference('SchemaChange', 'anyOf', 'DropPrimaryKey'); + contract.checkSpec( + contract.schema('BaseSchemaChange').discriminator.mapping.dropPrimaryKey === + '#/components/schemas/DropPrimaryKey', + 'BaseSchemaChange discriminator is missing dropPrimaryKey', + ); + const dropPrimaryKey = contract.requireProperties('DropPrimaryKey', ['action']); + contract.checkSpec( + dropPrimaryKey.action.const === 'dropPrimaryKey', + 'Schema DropPrimaryKey.action must be dropPrimaryKey', + ); + contract.checkSpec( + contract.schema('BaseInstant').discriminator.propertyName === 'type', + 'BaseInstant discriminator must use the JSON field type', + ); + + const updateViewComment = contract.requireProperties('UpdateViewComment', ['action', 'comment']); + contract.checkSpec( + !updateViewComment.key, + 'Schema UpdateViewComment must use comment instead of key', + ); + ['UpdateComment', 'UpdateViewComment', 'UpdateFunctionComment'].forEach((schemaName) => + requireNullableStringProperty(contract, schemaName, 'comment'), + ); + + const errorResourceTypes = + contract.requireProperties('ErrorResponse', ['resourceType']).resourceType.enum || []; + ['FUNCTION', 'DEFINITION'].forEach((resourceType) => { + contract.checkSpec( + errorResourceTypes.includes(resourceType), + `Schema ErrorResponse.resourceType is missing value: ${resourceType}`, + ); + }); + + requireArrayOfIdentifiers(contract, 'ListTablesGloballyResponse', 'tables'); + requireArrayOfIdentifiers(contract, 'ListViewsGloballyResponse', 'views'); + requireArrayOfIdentifiers(contract, 'ListFunctionsGloballyResponse', 'functions'); + contract.requireProperties('ListFunctionsGloballyResponse', ['nextPageToken']); + const getFunctionProperties = contract.requireProperties('GetFunctionResponse', ['uuid']); + contract.checkSpec( + getFunctionProperties.uuid.type === 'string', + 'Schema GetFunctionResponse.uuid must be a string', + ); + + ['GetDatabaseResponse', 'GetTableResponse', 'GetViewResponse', 'GetFunctionResponse'].forEach( + (schemaName) => contract.requireTypedIntegerProperties(schemaName, ['createdAt', 'updatedAt']), + ); + return contract.operations.size; } -[ - 'getConfig', - 'createDatabase', - 'getDatabase', - 'alterDatabase', - 'dropDatabase', - 'createTable', - 'getTable', - 'alterTable', - 'dropTable', -].forEach((operationId) => { - check(operationIds.has(operationId), `Missing provider-facing operationId: ${operationId}`); -}); - -requireProperties('ConfigResponse', ['defaults', 'overrides']); -requireProperties('CreateDatabaseRequest', ['name', 'options']); -requireProperties('AlterDatabaseRequest', ['removals', 'updates']); -requireProperties('CreateTableRequest', ['identifier', 'schema']); -requireProperties('AlterTableRequest', ['changes']); -requireProperties('Identifier', ['database', 'object']); -requireProperties('Schema', ['fields', 'partitionKeys', 'primaryKeys', 'options', 'comment']); -requireProperties('DataField', ['id', 'name', 'type', 'description', 'defaultValue']); - -requireSchemaReference('DataType', 'oneOf', 'VectorType'); -requireProperties('VectorType', ['type', 'element', 'length']); -requireSchemaReference('SchemaChange', 'anyOf', 'DropPrimaryKey'); -check( - schema('BaseSchemaChange').discriminator.mapping.dropPrimaryKey === - '#/components/schemas/DropPrimaryKey', - 'BaseSchemaChange discriminator is missing dropPrimaryKey', -); -const dropPrimaryKey = requireProperties('DropPrimaryKey', ['action']); -check( - dropPrimaryKey.action.const === 'dropPrimaryKey', - 'Schema DropPrimaryKey.action must be dropPrimaryKey', -); -check( - schema('BaseInstant').discriminator.propertyName === 'type', - 'BaseInstant discriminator must use the JSON field type', -); +function validateManagementOpenApi() { + const contract = validateCommon('rest-management-open-api.yaml'); + const operationIds = [ + 'listPermissions', + 'grantPermission', + 'revokePermission', + 'listTablePolicies', + 'createTablePolicy', + 'dropTablePolicy', + ]; + const resourcePaths = [ + '/v1/{prefix}/permissions', + '/v1/{prefix}/permissions/grant', + '/v1/{prefix}/permissions/revoke', + '/v1/{prefix}/databases/{database}/tables/{table}/policies', + ]; -const updateViewComment = requireProperties('UpdateViewComment', ['action', 'comment']); -check(!updateViewComment.key, 'Schema UpdateViewComment must use comment instead of key'); -['UpdateComment', 'UpdateViewComment', 'UpdateFunctionComment'].forEach((schemaName) => - requireNullableStringProperty(schemaName, 'comment'), -); + resourcePaths.forEach((resourcePath) => + contract.checkSpec( + contract.spec.paths[resourcePath], + `Missing management path: ${resourcePath}`, + ), + ); + [ + '/v1/{prefix}/policies', + '/v1/{prefix}/databases/{database}/policies', + '/v1/{prefix}/databases/{database}/tables/{table}/policies/{policyName}', + ].forEach((resourcePath) => + contract.checkSpec( + !contract.spec.paths[resourcePath], + `Policies must not be attachable outside tables: ${resourcePath}`, + ), + ); + operationIds.forEach(contract.requireOperation); + ['listPermissions', 'grantPermission', 'revokePermission'].forEach((operationId) => + contract.requireResponses(operationId, [ + '200', + '400', + '401', + '403', + '404', + '429', + '500', + '503', + ]), + ); + contract.requireResponses('grantPermission', ['409']); + ['listTablePolicies', 'createTablePolicy'].forEach((operationId) => + contract.requireResponses(operationId, [ + '200', + '400', + '401', + '403', + '404', + '429', + '500', + '503', + ]), + ); + contract.requireResponses('dropTablePolicy', [ + '200', + '400', + '401', + '403', + '404', + '429', + '500', + '503', + ]); + contract.requireResponses('createTablePolicy', ['409']); + contract.checkSpec( + contract.spec.info.version === '1.0' && + contract.spec.info.description.toLowerCase().includes('experimental'), + 'The management contract must be versioned 1.0 and marked experimental', + ); + contract.checkSpec( + !Object.prototype.hasOwnProperty.call(contract.spec, 'security'), + 'The management contract must not require one deployment-specific authentication scheme', + ); -const errorResourceTypes = requireProperties('ErrorResponse', ['resourceType']).resourceType.enum || []; -['FUNCTION', 'DEFINITION'].forEach((resourceType) => { - check( - errorResourceTypes.includes(resourceType), - `Schema ErrorResponse.resourceType is missing value: ${resourceType}`, + const assignmentFields = ['resource', 'access', 'principal', 'columns', 'expireTime']; + contract.requireProperties('PermissionAssignment', assignmentFields); + contract.requireRequiredProperties('PermissionAssignment', ['resource', 'access', 'principal']); + const grantProperties = contract.requireProperties('GrantPermissionRequest', [ + 'resource', + 'access', + 'principal', + 'columns', + 'expireTime', + ]); + contract.requireRequiredProperties('GrantPermissionRequest', [ + 'resource', + 'access', + 'principal', + ]); + ['policy', 'grantOption'].forEach((field) => + contract.checkSpec( + !grantProperties[field], + `Schema GrantPermissionRequest must omit field: ${field}`, + ), + ); + const revokeProperties = contract.requireProperties('RevokePermissionRequest', [ + 'resource', + 'access', + 'principal', + ]); + contract.requireRequiredProperties('RevokePermissionRequest', [ + 'resource', + 'access', + 'principal', + ]); + ['expireTime', 'columns', 'policy', 'grantOption'].forEach((field) => + contract.checkSpec( + !revokeProperties[field], + `Schema RevokePermissionRequest must omit field: ${field}`, + ), ); -}); -requireArrayOfIdentifiers('ListTablesGloballyResponse', 'tables'); -requireArrayOfIdentifiers('ListViewsGloballyResponse', 'views'); -requireArrayOfIdentifiers('ListFunctionsGloballyResponse', 'functions'); -requireProperties('ListFunctionsGloballyResponse', ['nextPageToken']); -const getFunctionProperties = requireProperties('GetFunctionResponse', ['uuid']); -check(getFunctionProperties.uuid.type === 'string', 'Schema GetFunctionResponse.uuid must be a string'); + const permissionList = contract.requireProperties('ListPermissionsResponse', [ + 'permissions', + 'nextPageToken', + ]); + contract.checkSpec( + permissionList.permissions.type === 'array' && + permissionList.permissions.items.$ref === '#/components/schemas/PermissionAssignment', + 'ListPermissionsResponse.permissions must contain PermissionAssignment values', + ); + contract.requireRequiredProperties('ListPermissionsResponse', ['permissions']); -['GetDatabaseResponse', 'GetTableResponse', 'GetViewResponse', 'GetFunctionResponse'].forEach( - (schemaName) => requireTypedIntegerProperties(schemaName, ['createdAt', 'updatedAt']), -); + const principal = contract.schema('Principal'); + contract.checkSpec( + principal.type === 'string' && principal.minLength === 1 && principal.maxLength === 128, + 'Principal must be a non-empty string that fits the 128-character persistence identity', + ); + contract.checkSpec( + !contract.spec.components.schemas.PrincipalRef && + !contract.spec.components.schemas.PrincipalType && + !contract.spec.components.parameters.PrincipalTypeQuery, + 'Principal must not expose a separate reference object or type', + ); + ['PermissionAssignment', 'GrantPermissionRequest', 'RevokePermissionRequest'].forEach( + (schemaName) => + contract.checkSpec( + contract.requireProperties(schemaName, ['principal']).principal.$ref === + '#/components/schemas/Principal', + `Schema ${schemaName}.principal must reference Principal`, + ), + ); + contract.checkSpec( + contract.spec.components.parameters.PrincipalQuery.schema.$ref === + '#/components/schemas/Principal', + 'PrincipalQuery must reference Principal', + ); + requireExactEnum(contract, 'ResourceType', [ + 'CATALOG', + 'CATALOG_ALL', + 'DATABASE', + 'DATABASE_ALL', + 'TABLE', + 'COLUMN', + 'FUNCTION', + 'VIEW', + ]); + requireExactEnum(contract, 'PolicyType', ['ROW_FILTER', 'COLUMN_MASKING']); + + const permissionAccess = contract.schema('PermissionAccess'); + const expectedAccesses = [ + 'ALL', + 'CREATEDATABASE', + 'DESCRIBE', + 'ALTER', + 'DROP', + 'CREATETABLE', + 'CREATEFUNCTION', + 'CREATEVIEW', + 'LIST', + 'SELECT', + 'UPDATE', + 'GRANT', + ]; + contract.checkSpec( + permissionAccess.type === 'string' && + permissionAccess.enum.length === expectedAccesses.length && + expectedAccesses.every((access) => permissionAccess.enum.includes(access)), + 'PermissionAccess must define the complete data access enum', + ); + contract.checkSpec( + permissionAccess.maxLength === 32, + 'PermissionAccess must fit the 32-character persistence field', + ); + ['PermissionAssignment', 'GrantPermissionRequest', 'RevokePermissionRequest'].forEach( + (schemaName) => + contract.checkSpec( + contract.requireProperties(schemaName, ['access']).access.$ref === + '#/components/schemas/PermissionAccess', + `Schema ${schemaName}.access must reference PermissionAccess`, + ), + ); + contract.checkSpec( + contract.spec.components.parameters.AccessQuery.schema.$ref === + '#/components/schemas/PermissionAccess', + 'AccessQuery must reference PermissionAccess', + ); + contract.requireSchemaReference('PermissionResource', 'oneOf', 'ColumnResource'); + contract.requireSchemaReference('PermissionResource', 'oneOf', 'CatalogAllResource'); + contract.requireSchemaReference('PermissionResource', 'oneOf', 'DatabaseAllResource'); + const catalogAllResource = contract.requireProperties('CatalogAllResource', ['type']); + contract.requireRequiredProperties('CatalogAllResource', ['type']); + contract.checkSpec( + catalogAllResource.type.const === 'CATALOG_ALL', + 'CatalogAllResource must use the CATALOG_ALL discriminator', + ); + const databaseAllResource = contract.requireProperties('DatabaseAllResource', [ + 'type', + 'database', + ]); + contract.requireRequiredProperties('DatabaseAllResource', ['type', 'database']); + contract.checkSpec( + databaseAllResource.type.const === 'DATABASE_ALL', + 'DatabaseAllResource must use the DATABASE_ALL discriminator', + ); + const columnResource = contract.requireProperties('ColumnResource', [ + 'type', + 'database', + 'table', + ]); + contract.requireRequiredProperties('ColumnResource', ['type', 'database', 'table']); + contract.checkSpec( + columnResource.type.const === 'COLUMN', + 'ColumnResource must use the COLUMN discriminator', + ); + const permissionColumns = contract.requireProperties('PermissionColumns', [ + 'columnNames', + 'excludedColumnNames', + ]); + const permissionColumnsDescription = contract + .schema('PermissionColumns') + .description.toLowerCase() + .replace(/\s+/g, ' '); + ['intersected', 'fails the query', 'query authorization'].forEach((phrase) => + contract.checkSpec( + permissionColumnsDescription.includes(phrase), + `PermissionColumns semantics are missing: ${phrase}`, + ), + ); + ['columnNames', 'excludedColumnNames'].forEach((field) => { + const definition = permissionColumns[field]; + contract.checkSpec( + definition.type === 'array' && + definition.minItems === 1 && + definition.uniqueItems === true && + definition.items.type === 'string' && + definition.items.minLength === 1, + `PermissionColumns.${field} must be a non-empty unique string array`, + ); + }); + const columnAlternatives = contract.schema('PermissionColumns').oneOf || []; + ['columnNames', 'excludedColumnNames'].forEach((field) => + contract.checkSpec( + columnAlternatives.some( + (alternative) => + alternative.required && + alternative.required.length === 1 && + alternative.required[0] === field, + ), + `PermissionColumns must define the ${field} alternative`, + ), + ); + ['PermissionAssignment', 'GrantPermissionRequest'].forEach((schemaName) => { + const properties = contract.requireProperties(schemaName, ['columns']); + contract.checkSpec( + properties.columns.$ref === '#/components/schemas/PermissionColumns', + `${schemaName}.columns must reference PermissionColumns`, + ); + contract.requireSchemaReference(schemaName, 'allOf', 'ColumnAssignmentConstraint'); + }); + ['DatabaseQuery', 'TableQuery', 'FunctionQuery', 'ViewQuery'].forEach((parameterName) => { + const locatorQuery = contract.spec.components.parameters[parameterName]; + contract.checkSpec( + locatorQuery.schema.type === 'string' && locatorQuery.schema.minLength === 1, + `${parameterName} must be a non-empty string`, + ); + }); + + ['TooManyRequests', 'ServiceUnavailable'].forEach((responseName) => { + const response = contract.spec.components.responses[responseName]; + contract.checkSpec(response, `Missing reusable response: ${responseName}`); + contract.checkSpec( + response.headers['Retry-After'].$ref === '#/components/headers/RetryAfter', + `Response ${responseName} must expose the optional Retry-After header`, + ); + }); + contract.checkSpec( + contract.spec.components.headers.RetryAfter.schema.type === 'string', + 'RetryAfter must allow HTTP delta-seconds or an HTTP date as a string', + ); + + const assignmentExpiry = contract.requireProperties('PermissionAssignment', ['expireTime']) + .expireTime.description.toLowerCase(); + const grantExpiry = contract.requireProperties('GrantPermissionRequest', ['expireTime']) + .expireTime.description.toLowerCase(); + [assignmentExpiry, grantExpiry].forEach((description) => { + contract.checkSpec( + description.includes('exclusive') && + description.includes('server clock') && + description.includes('millisecond') && + description.includes('must not authorize'), + 'expireTime must define exclusive millisecond server-clock authorization semantics', + ); + }); + const resourceDescription = contract.schema('PermissionResource').description.toLowerCase(); + ['stable internal resource identity', 'renaming', 'dropping', 'recreating'].forEach((phrase) => + contract.checkSpec( + resourceDescription.includes(phrase), + `PermissionResource lifecycle is missing: ${phrase}`, + ), + ); + const policyDescription = contract.schema('DataPolicy').description.toLowerCase(); + ['logical and', 'same column', 'fail closed'].forEach((phrase) => + contract.checkSpec( + policyDescription.includes(phrase), + `DataPolicy composition is missing: ${phrase}`, + ), + ); -console.log(`Validated REST OpenAPI contract with ${operationIds.size} operations.`); + contract.requireSchemaReference('PolicyRequest', 'oneOf', 'RowFilterPolicyRequest'); + contract.requireSchemaReference('PolicyRequest', 'oneOf', 'ColumnMaskPolicyRequest'); + contract.requireSchemaReference('DataPolicy', 'oneOf', 'RowFilterDataPolicy'); + contract.requireSchemaReference('DataPolicy', 'oneOf', 'ColumnMaskDataPolicy'); + const rowFilter = contract.requireProperties('RowFilter', ['predicate']); + const columnMask = contract.requireProperties('ColumnMask', ['onColumn', 'transform']); + contract.requireRequiredProperties('RowFilter', ['predicate']); + contract.requireRequiredProperties('ColumnMask', ['onColumn', 'transform']); + [rowFilter.predicate, columnMask.transform].forEach((definition) => { + contract.checkSpec( + definition.type === 'string' && + definition.minLength === 1 && + definition['x-maxUtf8Bytes'] === 61440 && + definition.contentMediaType === 'application/json', + 'Policy definitions must be bounded non-empty JSON strings', + ); + }); + ['RowFilterPolicyRequest', 'ColumnMaskPolicyRequest'].forEach((schemaName) => { + const properties = contract.requireProperties(schemaName, ['principal']); + contract.checkSpec( + properties.principal.$ref === '#/components/schemas/Principal', + `${schemaName}.principal must reference Principal`, + ); + contract.checkSpec( + !properties.type && + !properties.resource && + !properties.name && + !properties.toPrincipals && + !properties.exceptPrincipals, + `${schemaName} must expose only one principal and no path identity`, + ); + contract.requireRequiredProperties(schemaName, ['principal']); + }); + contract.requireProperties('RowFilterPolicyRequest', ['rowFilter']); + contract.requireProperties('ColumnMaskPolicyRequest', ['columnMask']); + contract.requireRequiredProperties('RowFilterPolicyRequest', ['rowFilter']); + contract.requireRequiredProperties('ColumnMaskPolicyRequest', ['columnMask']); + contract.requireSchemaReference('DropPolicyRequest', 'oneOf', 'RowFilterPolicyIdentity'); + contract.requireSchemaReference('DropPolicyRequest', 'oneOf', 'ColumnMaskPolicyIdentity'); + contract.requireRequiredProperties('RowFilterPolicyIdentity', ['type', 'principal']); + contract.requireRequiredProperties('ColumnMaskPolicyIdentity', [ + 'type', + 'principal', + 'column', + ]); + const tablePolicyResource = contract.requireProperties('TablePolicyResource', [ + 'type', + 'database', + 'table', + ]); + contract.checkSpec( + tablePolicyResource.type.const === 'TABLE', + 'Data policies must use a TABLE attachment resource', + ); + const policyList = contract.requireProperties('ListPoliciesResponse', [ + 'policies', + 'nextPageToken', + ]); + contract.checkSpec( + policyList.policies.items.$ref === '#/components/schemas/DataPolicy', + 'ListPoliciesResponse.policies must contain DataPolicy values', + ); + ['RowFilterDataPolicy', 'ColumnMaskDataPolicy'].forEach((schemaName) => { + const properties = contract.requireProperties(schemaName, ['resource', 'principal']); + contract.checkSpec( + properties.principal.$ref === '#/components/schemas/Principal', + `${schemaName}.principal must reference Principal`, + ); + contract.checkSpec( + !properties.name && !properties.toPrincipals && !properties.exceptPrincipals, + `${schemaName} must use a single principal identity`, + ); + }); + contract.requireProperties('ErrorResponse', [ + 'message', + 'resourceType', + 'resourceName', + 'code', + ]); + contract.requireRequiredProperties('ErrorResponse', ['message', 'code']); + + const permissionParameters = contract + .requireOperation('listPermissions') + .parameters.map((parameter) => + parameter.$ref ? parameter.$ref.split('/').pop() : parameter.name, + ); + [ + 'ResourceTypeQuery', + 'DatabaseQuery', + 'TableQuery', + 'FunctionQuery', + 'ViewQuery', + 'PrincipalQuery', + 'AccessQuery', + 'PageToken', + 'MaxResults', + ].forEach((name) => + contract.checkSpec( + permissionParameters.includes(name), + `Operation listPermissions is missing query parameter: ${name}`, + ), + ); + contract.checkSpec( + permissionParameters.length === 9, + 'Operation listPermissions must expose only exact-resource filters and pagination', + ); + contract.checkSpec( + contract.spec.components.parameters.ResourceTypeQuery.required === true, + 'Operation listPermissions must require resourceType', + ); + const policyParameters = contract + .requireOperation('listTablePolicies') + .parameters.map((parameter) => + parameter.$ref ? parameter.$ref.split('/').pop() : parameter.name, + ); + ['PolicyTypeQuery', 'PrincipalQuery', 'PolicyColumnQuery', 'PageToken', 'MaxResults'].forEach( + (name) => + contract.checkSpec( + policyParameters.includes(name), + `Operation listTablePolicies is missing query parameter: ${name}`, + ), + ); + contract.checkSpec( + !policyParameters.includes('PolicyNameQuery'), + 'Principal-scoped policies must not expose a policy-name filter', + ); + const tablePoliciesPath = + contract.spec.paths['/v1/{prefix}/databases/{database}/tables/{table}/policies']; + contract.checkSpec( + tablePoliciesPath.post && tablePoliciesPath.delete && !tablePoliciesPath.put, + 'Policy creation and deletion must use the table policy collection identity without PUT', + ); + + contract.checkSpec( + contract.operations.size === operationIds.length, + `The management contract must define exactly ${operationIds.length} operations`, + ); + resourcePaths.forEach((resourcePath) => + contract.checkSpec( + contract.spec.paths[resourcePath].parameters.some( + (parameter) => parameter.$ref === '#/components/parameters/Prefix', + ), + `Path ${resourcePath} must reuse components.parameters.Prefix`, + ), + ); + return contract.operations.size; +} + +const catalogOperationCount = validateCatalogOpenApi(); +const managementOperationCount = validateManagementOpenApi(); +console.log(`Validated REST Catalog OpenAPI contract with ${catalogOperationCount} operations.`); +console.log( + `Validated REST Management OpenAPI contract with ${managementOperationCount} operations.`, +); diff --git a/docs/sidebars.js b/docs/sidebars.js index cb70d4ad19fd..cd2b6b2d95f3 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -31,7 +31,8 @@ const sidebars = { "concepts/rest/dlf", "concepts/rest/tables", "concepts/rest/pvfs", - "concepts/rest/rest-api" + "concepts/rest/rest-api", + "concepts/rest/management-api" ] }, { diff --git a/docs/static/rest-management-open-api.yaml b/docs/static/rest-management-open-api.yaml new file mode 100644 index 000000000000..b30298a16e57 --- /dev/null +++ b/docs/static/rest-management-open-api.yaml @@ -0,0 +1,837 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +openapi: 3.1.1 +info: + title: Paimon REST Management API + version: "1.0" + description: | + Experimental control-plane extension for object and column permissions, table row filters, and + table column masks in one Paimon REST Catalog prefix. A data policy restricts an + already-authorized read; + it never grants SELECT by itself. Each policy is attached to one principal: a principal has at + most one row filter per table and at most one mask per table column. Policy creation is accepted + only when the target table exists, has `query-auth.enabled=true`, and the referenced principal, + serialized Paimon predicate or transform, and columns are valid. + Authentication follows the REST Catalog deployment configuration and is not fixed by this + extension. + Principal lifecycle, audit, and persistence remain server responsibilities. This contract may + evolve incompatibly while experimental. + license: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0.html +servers: + - url: http://localhost:8080 +paths: + /v1/{prefix}/permissions: + parameters: + - $ref: '#/components/parameters/Prefix' + get: + tags: [permission] + summary: List direct permission assignments on a resource or scope + operationId: listPermissions + description: | + Returns direct assignments attached to the requested exact resource or explicit descendant + scope. This operation does not synthesize assignments effective through CATALOG_ALL or + DATABASE_ALL. Expired assignments may remain visible until server cleanup, but must not + authorize access. Following the catalog pagination contract, an empty page terminates + pagination and therefore must not carry a continuation token. + parameters: + - $ref: '#/components/parameters/ResourceTypeQuery' + - $ref: '#/components/parameters/DatabaseQuery' + - $ref: '#/components/parameters/TableQuery' + - $ref: '#/components/parameters/FunctionQuery' + - $ref: '#/components/parameters/ViewQuery' + - $ref: '#/components/parameters/PrincipalQuery' + - $ref: '#/components/parameters/AccessQuery' + - $ref: '#/components/parameters/PageToken' + - $ref: '#/components/parameters/MaxResults' + responses: + '200': + description: Permission assignments in stable pagination order. + content: + application/json: + schema: + $ref: '#/components/schemas/ListPermissionsResponse' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '429': + $ref: '#/components/responses/TooManyRequests' + '500': + $ref: '#/components/responses/ServerError' + '503': + $ref: '#/components/responses/ServiceUnavailable' + /v1/{prefix}/permissions/grant: + parameters: + - $ref: '#/components/parameters/Prefix' + post: + tags: [permission] + summary: Grant or replace a permission assignment + operationId: grantPermission + description: | + Creates an assignment or replaces the assignment with the same `resource`, `access`, and + `principal`. For a `COLUMN` assignment this replaces the whole included or excluded column + range. The referenced resource, principal, and columns must exist. Servers bind assignments + to the resource lifecycle described by `PermissionResource`. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/GrantPermissionRequest' + responses: + '200': + description: Permission assignment created or replaced. + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '409': + $ref: '#/components/responses/Conflict' + '429': + $ref: '#/components/responses/TooManyRequests' + '500': + $ref: '#/components/responses/ServerError' + '503': + $ref: '#/components/responses/ServiceUnavailable' + /v1/{prefix}/permissions/revoke: + parameters: + - $ref: '#/components/parameters/Prefix' + post: + tags: [permission] + summary: Idempotently revoke a permission assignment + operationId: revokePermission + description: | + Makes the assignment absent. A 404 identifies a missing resource or principal, not an + already-absent assignment. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/RevokePermissionRequest' + responses: + '200': + description: Permission assignment is absent. + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '429': + $ref: '#/components/responses/TooManyRequests' + '500': + $ref: '#/components/responses/ServerError' + '503': + $ref: '#/components/responses/ServiceUnavailable' + + /v1/{prefix}/databases/{database}/tables/{table}/policies: + parameters: + - $ref: '#/components/parameters/Prefix' + - $ref: '#/components/parameters/Database' + - $ref: '#/components/parameters/Table' + get: + tags: [policy] + summary: List policies attached directly to a table + operationId: listTablePolicies + description: | + Policies are attached directly to the requested table. Following the catalog pagination + contract, an empty page terminates pagination and therefore must not carry a continuation + token. + parameters: + - $ref: '#/components/parameters/PolicyTypeQuery' + - $ref: '#/components/parameters/PrincipalQuery' + - $ref: '#/components/parameters/PolicyColumnQuery' + - $ref: '#/components/parameters/PageToken' + - $ref: '#/components/parameters/MaxResults' + responses: + '200': + description: Table policies in stable pagination order. + content: + application/json: + schema: + $ref: '#/components/schemas/ListPoliciesResponse' + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '429': + $ref: '#/components/responses/TooManyRequests' + '500': + $ref: '#/components/responses/ServerError' + '503': + $ref: '#/components/responses/ServiceUnavailable' + post: + tags: [policy] + summary: Create a table policy + operationId: createTablePolicy + description: | + Fails with 409 when the same principal already has a row filter or a mask on the same + column, or when the table has not enabled query authorization. The principal, policy + predicate or transform, protected column, and referenced fields are validated before + persistence. + requestBody: + $ref: '#/components/requestBodies/PolicyRequest' + responses: + '200': + description: Policy created. + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '409': + $ref: '#/components/responses/Conflict' + '429': + $ref: '#/components/responses/TooManyRequests' + '500': + $ref: '#/components/responses/ServerError' + '503': + $ref: '#/components/responses/ServiceUnavailable' + delete: + tags: [policy] + summary: Drop a table policy + operationId: dropTablePolicy + description: Returns 404 when the policy is absent; clients may expose idempotency explicitly. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/DropPolicyRequest' + responses: + '200': + description: Policy dropped. + '400': + $ref: '#/components/responses/BadRequest' + '401': + $ref: '#/components/responses/Unauthorized' + '403': + $ref: '#/components/responses/Forbidden' + '404': + $ref: '#/components/responses/NotFound' + '429': + $ref: '#/components/responses/TooManyRequests' + '500': + $ref: '#/components/responses/ServerError' + '503': + $ref: '#/components/responses/ServiceUnavailable' + +components: + securitySchemes: + BearerAuth: + type: http + scheme: bearer + parameters: + Prefix: + name: prefix + in: path + required: true + description: Opaque REST catalog prefix returned by the catalog config endpoint. + schema: + type: string + minLength: 1 + Database: + name: database + in: path + required: true + schema: + type: string + minLength: 1 + Table: + name: table + in: path + required: true + schema: + type: string + minLength: 1 + ResourceTypeQuery: + name: resourceType + in: query + required: true + schema: + $ref: '#/components/schemas/ResourceType' + DatabaseQuery: + name: database + in: query + schema: + type: string + minLength: 1 + TableQuery: + name: table + in: query + schema: + type: string + minLength: 1 + FunctionQuery: + name: function + in: query + schema: + type: string + minLength: 1 + ViewQuery: + name: view + in: query + schema: + type: string + minLength: 1 + PrincipalQuery: + name: principal + in: query + description: Exact opaque principal identifier. + schema: + $ref: '#/components/schemas/Principal' + AccessQuery: + name: access + in: query + schema: + $ref: '#/components/schemas/PermissionAccess' + PolicyColumnQuery: + name: column + in: query + description: Valid only together with `type=COLUMN_MASKING`. + schema: + type: string + minLength: 1 + PolicyTypeQuery: + name: type + in: query + schema: + $ref: '#/components/schemas/PolicyType' + PageToken: + name: pageToken + in: query + description: Opaque continuation token returned by the preceding response. + schema: + type: string + MaxResults: + name: maxResults + in: query + schema: + type: integer + minimum: 1 + maximum: 1000 + + requestBodies: + PolicyRequest: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/PolicyRequest' + + responses: + BadRequest: + description: Invalid request shape, resource identity, predicate, transform, or column. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + Unauthorized: + description: Missing or invalid authentication. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + Forbidden: + description: Caller cannot manage the target resource. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + NotFound: + description: Referenced resource, principal, or policy does not exist. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + Conflict: + description: Existing policy conflict or query authorization is not enabled. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + TooManyRequests: + description: Request rate limit exceeded; retry only according to server guidance. + headers: + Retry-After: + $ref: '#/components/headers/RetryAfter' + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + ServerError: + description: Unexpected server error. + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + ServiceUnavailable: + description: Management service is temporarily unavailable. + headers: + Retry-After: + $ref: '#/components/headers/RetryAfter' + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorResponse' + + headers: + RetryAfter: + description: Delay before retrying, as HTTP delta-seconds or an HTTP date. + schema: + type: string + + schemas: + ResourceType: + type: string + enum: [CATALOG, CATALOG_ALL, DATABASE, DATABASE_ALL, TABLE, COLUMN, FUNCTION, VIEW] + PermissionAccess: + description: | + Canonical data access value. The REST wire format uses upper case and creation accesses + follow their persisted names without underscores. UPDATE covers table writes, including + inserts and deletes. SELECT also covers function use. GRANT allows granting and revoking + assignments on the resource. The server validates applicability to the resource type. + type: string + maxLength: 32 + enum: + - ALL + - CREATEDATABASE + - DESCRIBE + - ALTER + - DROP + - CREATETABLE + - CREATEFUNCTION + - CREATEVIEW + - LIST + - SELECT + - UPDATE + - GRANT + PolicyType: + type: string + enum: [ROW_FILTER, COLUMN_MASKING] + Principal: + type: string + minLength: 1 + maxLength: 128 + description: | + Opaque, canonical, globally unique identifier in the server principal namespace. Type and + membership resolution are server responsibilities. + PermissionResource: + description: | + Exact catalog-scoped wire locator or explicit descendant scope; fields not selected by type + are invalid. CATALOG_ALL covers descendants of the configured catalog, while DATABASE_ALL + covers descendants of the named database. Servers must bind stored direct assignments to a + stable internal resource identity. Renaming a database, table, function, or view retains its + direct assignments and responses use the new locator. Dropping a resource removes its direct + assignments, and recreating the same locator must not inherit them. + oneOf: + - $ref: '#/components/schemas/CatalogResource' + - $ref: '#/components/schemas/CatalogAllResource' + - $ref: '#/components/schemas/DatabaseResource' + - $ref: '#/components/schemas/DatabaseAllResource' + - $ref: '#/components/schemas/TableResource' + - $ref: '#/components/schemas/ColumnResource' + - $ref: '#/components/schemas/FunctionResource' + - $ref: '#/components/schemas/ViewResource' + discriminator: + propertyName: type + mapping: + CATALOG: '#/components/schemas/CatalogResource' + CATALOG_ALL: '#/components/schemas/CatalogAllResource' + DATABASE: '#/components/schemas/DatabaseResource' + DATABASE_ALL: '#/components/schemas/DatabaseAllResource' + TABLE: '#/components/schemas/TableResource' + COLUMN: '#/components/schemas/ColumnResource' + FUNCTION: '#/components/schemas/FunctionResource' + VIEW: '#/components/schemas/ViewResource' + CatalogResource: + type: object + additionalProperties: false + required: [type] + properties: + type: + const: CATALOG + CatalogAllResource: + type: object + additionalProperties: false + description: Explicit scope covering database, table, view, and function descendants. + required: [type] + properties: + type: + const: CATALOG_ALL + DatabaseResource: + type: object + additionalProperties: false + required: [type, database] + properties: + type: + const: DATABASE + database: + type: string + minLength: 1 + DatabaseAllResource: + type: object + additionalProperties: false + description: Explicit scope covering table, view, and function descendants of one database. + required: [type, database] + properties: + type: + const: DATABASE_ALL + database: + type: string + minLength: 1 + TableResource: + type: object + additionalProperties: false + required: [type, database, table] + properties: + type: + const: TABLE + database: + type: string + minLength: 1 + table: + type: string + minLength: 1 + ColumnResource: + type: object + additionalProperties: false + description: | + Column permission target. The resource identity is the containing table; the mutable column + range is carried by `PermissionAssignment.columns` and is not part of the identity. + required: [type, database, table] + properties: + type: + const: COLUMN + database: + type: string + minLength: 1 + table: + type: string + minLength: 1 + FunctionResource: + type: object + additionalProperties: false + required: [type, database, function] + properties: + type: + const: FUNCTION + database: + type: string + minLength: 1 + function: + type: string + minLength: 1 + ViewResource: + type: object + additionalProperties: false + required: [type, database, view] + properties: + type: + const: VIEW + database: + type: string + minLength: 1 + view: + type: string + minLength: 1 + PermissionAssignment: + type: object + additionalProperties: false + description: | + Direct assignment identity is `resource`, `access`, and `principal`. CATALOG_ALL and + DATABASE_ALL are explicit descendant scopes rather than computed effective assignments. + `columns` is required only for a COLUMN assignment and is replaced as one value when the + same identity is granted again. + required: [resource, access, principal] + properties: + resource: + $ref: '#/components/schemas/PermissionResource' + access: + $ref: '#/components/schemas/PermissionAccess' + principal: + $ref: '#/components/schemas/Principal' + columns: + $ref: '#/components/schemas/PermissionColumns' + expireTime: + type: string + format: date-time + description: | + Exclusive authorization upper bound evaluated using the server clock. At + `now >= expireTime` this assignment must not authorize access. An expired record may + remain listable until cleanup. The value must have at most millisecond precision. + allOf: + - $ref: '#/components/schemas/ColumnAssignmentConstraint' + GrantPermissionRequest: + type: object + additionalProperties: false + required: [resource, access, principal] + properties: + resource: + $ref: '#/components/schemas/PermissionResource' + access: + $ref: '#/components/schemas/PermissionAccess' + principal: + $ref: '#/components/schemas/Principal' + columns: + $ref: '#/components/schemas/PermissionColumns' + expireTime: + type: string + format: date-time + description: | + Exclusive authorization upper bound evaluated using the server clock. At + `now >= expireTime` the assignment must not authorize access. The value must have at + most millisecond precision. + allOf: + - $ref: '#/components/schemas/ColumnAssignmentConstraint' + PermissionColumns: + type: object + additionalProperties: false + description: | + Exactly one non-empty list of top-level table column names. `columnNames` is an allowlist; + `excludedColumnNames` grants every current table column except the listed denylist. Column + names must exist when granted. An allowlist denies columns added later, while a denylist + allows columns added later. All applicable column ranges are intersected, and selecting any + column outside the effective range fails the query. The target table must enforce query + authorization before the grant becomes visible. + properties: + columnNames: + type: array + minItems: 1 + uniqueItems: true + items: + type: string + minLength: 1 + excludedColumnNames: + type: array + minItems: 1 + uniqueItems: true + items: + type: string + minLength: 1 + oneOf: + - required: [columnNames] + - required: [excludedColumnNames] + ColumnAssignmentConstraint: + if: + properties: + resource: + type: object + required: [type] + properties: + type: + const: COLUMN + required: [resource] + then: + required: [columns] + else: + not: + required: [columns] + RevokePermissionRequest: + type: object + additionalProperties: false + required: [resource, access, principal] + properties: + resource: + $ref: '#/components/schemas/PermissionResource' + access: + $ref: '#/components/schemas/PermissionAccess' + principal: + $ref: '#/components/schemas/Principal' + ListPermissionsResponse: + type: object + additionalProperties: false + required: [permissions] + properties: + permissions: + type: array + items: + $ref: '#/components/schemas/PermissionAssignment' + nextPageToken: + type: string + + RowFilter: + type: object + additionalProperties: false + required: [predicate] + properties: + predicate: + type: string + minLength: 1 + x-maxUtf8Bytes: 61440 + contentMediaType: application/json + description: | + JSON serialization of one Paimon `Predicate`, using the same representation as one + entry in `AuthTableQueryResponse.filter`. The UTF-8 representation must not exceed + 60 KiB. The server must deserialize, validate against the target table schema, and + canonicalize it when the policy is created. + ColumnMask: + type: object + additionalProperties: false + required: [onColumn, transform] + properties: + onColumn: + type: string + minLength: 1 + transform: + type: string + minLength: 1 + x-maxUtf8Bytes: 61440 + contentMediaType: application/json + description: | + JSON serialization of one Paimon `Transform`, using the same representation as the + value for `onColumn` in `AuthTableQueryResponse.columnMasking`. The UTF-8 + representation must not exceed 60 KiB. The server must deserialize it, validate all + field references and the output type against the target table schema, and canonicalize + it when the policy is created. + RowFilterPolicyRequest: + type: object + additionalProperties: false + required: [rowFilter, principal] + properties: + rowFilter: + $ref: '#/components/schemas/RowFilter' + principal: + $ref: '#/components/schemas/Principal' + ColumnMaskPolicyRequest: + type: object + additionalProperties: false + required: [columnMask, principal] + properties: + columnMask: + $ref: '#/components/schemas/ColumnMask' + principal: + $ref: '#/components/schemas/Principal' + PolicyRequest: + description: Exactly one typed policy definition for one principal is allowed. + oneOf: + - $ref: '#/components/schemas/RowFilterPolicyRequest' + - $ref: '#/components/schemas/ColumnMaskPolicyRequest' + RowFilterPolicyIdentity: + type: object + additionalProperties: false + required: [type, principal] + properties: + type: + const: ROW_FILTER + principal: + $ref: '#/components/schemas/Principal' + ColumnMaskPolicyIdentity: + type: object + additionalProperties: false + required: [type, principal, column] + properties: + type: + const: COLUMN_MASKING + principal: + $ref: '#/components/schemas/Principal' + column: + type: string + minLength: 1 + DropPolicyRequest: + description: Exact principal policy identity on the table named by the request path. + oneOf: + - $ref: '#/components/schemas/RowFilterPolicyIdentity' + - $ref: '#/components/schemas/ColumnMaskPolicyIdentity' + TablePolicyResource: + type: object + additionalProperties: false + required: [type, database, table] + properties: + type: + const: TABLE + database: + type: string + minLength: 1 + table: + type: string + minLength: 1 + RowFilterDataPolicy: + type: object + additionalProperties: false + required: [resource, rowFilter, principal] + properties: + resource: + $ref: '#/components/schemas/TablePolicyResource' + rowFilter: + $ref: '#/components/schemas/RowFilter' + principal: + $ref: '#/components/schemas/Principal' + ColumnMaskDataPolicy: + type: object + additionalProperties: false + required: [resource, columnMask, principal] + properties: + resource: + $ref: '#/components/schemas/TablePolicyResource' + columnMask: + $ref: '#/components/schemas/ColumnMask' + principal: + $ref: '#/components/schemas/Principal' + DataPolicy: + description: | + A policy applies when its principal is effective for the caller. All applicable row filters + must be combined with logical AND. More than one applicable column mask for the same column + must fail closed. An invalid, unsupported, or schema-incompatible predicate or transform + must also fail closed rather than omit a restriction. + oneOf: + - $ref: '#/components/schemas/RowFilterDataPolicy' + - $ref: '#/components/schemas/ColumnMaskDataPolicy' + ListPoliciesResponse: + type: object + additionalProperties: false + required: [policies] + properties: + policies: + type: array + items: + $ref: '#/components/schemas/DataPolicy' + nextPageToken: + type: string + ErrorResponse: + type: object + additionalProperties: true + required: [message, code] + properties: + resourceType: + type: string + resourceName: + type: string + message: + type: string + code: + type: integer diff --git a/paimon-api/src/main/java/org/apache/paimon/management/ColumnMask.java b/paimon-api/src/main/java/org/apache/paimon/management/ColumnMask.java new file mode 100644 index 000000000000..c2dbadbbdb7b --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/management/ColumnMask.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.management; + +import org.apache.paimon.annotation.Experimental; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.beans.ConstructorProperties; +import java.nio.charset.StandardCharsets; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Protected column and serialized Paimon transform for a column-mask policy. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class ColumnMask { + + public static final int MAX_TRANSFORM_BYTES = 60 * 1024; + + private static final String FIELD_ON_COLUMN = "onColumn"; + private static final String FIELD_TRANSFORM = "transform"; + + @JsonProperty(FIELD_ON_COLUMN) + private final String onColumn; + + @JsonProperty(FIELD_TRANSFORM) + private final String transform; + + @JsonCreator + @ConstructorProperties({FIELD_ON_COLUMN, FIELD_TRANSFORM}) + public ColumnMask( + @JsonProperty(FIELD_ON_COLUMN) String onColumn, + @JsonProperty(FIELD_TRANSFORM) String transform) { + checkArgument(!isBlank(onColumn), "onColumn cannot be empty."); + checkArgument(!isBlank(transform), "transform cannot be empty."); + checkArgument( + transform.getBytes(StandardCharsets.UTF_8).length <= MAX_TRANSFORM_BYTES, + "transform must not exceed %s UTF-8 bytes.", + MAX_TRANSFORM_BYTES); + this.onColumn = onColumn; + this.transform = transform; + } + + @JsonGetter(FIELD_ON_COLUMN) + public String getOnColumn() { + return onColumn; + } + + @JsonGetter(FIELD_TRANSFORM) + public String getTransform() { + return transform; + } + + private static boolean isBlank(String value) { + return value == null || value.trim().isEmpty(); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/management/DataPolicy.java b/paimon-api/src/main/java/org/apache/paimon/management/DataPolicy.java new file mode 100644 index 000000000000..08fe3fb0357d --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/management/DataPolicy.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.management; + +import org.apache.paimon.annotation.Experimental; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.beans.ConstructorProperties; + +import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkNotNull; + +/** + * Principal-scoped row-filter or column-mask policy attached to one table. + * + *

A principal has at most one row filter per table and at most one mask per table column. When + * policies are enforced, applicable row filters are combined with logical AND and multiple + * effective masks for one column fail closed. Invalid predicates or transforms also fail closed. + */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class DataPolicy { + + private static final String FIELD_RESOURCE = "resource"; + private static final String FIELD_ROW_FILTER = "rowFilter"; + private static final String FIELD_COLUMN_MASK = "columnMask"; + private static final String FIELD_PRINCIPAL = "principal"; + + @JsonProperty(FIELD_RESOURCE) + private final PermissionResource resource; + + @Nullable + @JsonProperty(FIELD_ROW_FILTER) + @JsonInclude(JsonInclude.Include.NON_NULL) + private final RowFilter rowFilter; + + @Nullable + @JsonProperty(FIELD_COLUMN_MASK) + @JsonInclude(JsonInclude.Include.NON_NULL) + private final ColumnMask columnMask; + + @JsonProperty(FIELD_PRINCIPAL) + private final String principal; + + @JsonCreator + @ConstructorProperties({FIELD_RESOURCE, FIELD_ROW_FILTER, FIELD_COLUMN_MASK, FIELD_PRINCIPAL}) + public DataPolicy( + @JsonProperty(FIELD_RESOURCE) PermissionResource resource, + @Nullable @JsonProperty(FIELD_ROW_FILTER) RowFilter rowFilter, + @Nullable @JsonProperty(FIELD_COLUMN_MASK) ColumnMask columnMask, + @JsonProperty(FIELD_PRINCIPAL) String principal) { + this.resource = checkNotNull(resource, "resource cannot be null"); + resource.validatePolicyAttachment(); + checkArgument( + (rowFilter == null) != (columnMask == null), + "A policy must contain exactly one of rowFilter and columnMask."); + this.rowFilter = rowFilter; + this.columnMask = columnMask; + this.principal = PermissionAssignment.validatePrincipal(principal); + } + + public static DataPolicy rowFilter( + PermissionResource resource, RowFilter rowFilter, String principal) { + return new DataPolicy(resource, rowFilter, null, principal); + } + + public static DataPolicy columnMask( + PermissionResource resource, ColumnMask columnMask, String principal) { + return new DataPolicy(resource, null, columnMask, principal); + } + + @JsonGetter(FIELD_RESOURCE) + public PermissionResource getResource() { + return resource; + } + + @Nullable + @JsonGetter(FIELD_ROW_FILTER) + public RowFilter getRowFilter() { + return rowFilter; + } + + @Nullable + @JsonGetter(FIELD_COLUMN_MASK) + public ColumnMask getColumnMask() { + return columnMask; + } + + public PolicyType type() { + return rowFilter == null ? PolicyType.COLUMN_MASKING : PolicyType.ROW_FILTER; + } + + @JsonGetter(FIELD_PRINCIPAL) + public String getPrincipal() { + return principal; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/management/ListPermissionsRequest.java b/paimon-api/src/main/java/org/apache/paimon/management/ListPermissionsRequest.java new file mode 100644 index 000000000000..63e78dd884b9 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/management/ListPermissionsRequest.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.management; + +import org.apache.paimon.annotation.Experimental; + +import javax.annotation.Nullable; + +import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkNotNull; + +/** Exact resource or scope, principal, and pagination filters for permission assignments. */ +@Experimental +public class ListPermissionsRequest { + + public static final int MAX_PAGE_SIZE = 1000; + + private final PermissionResource resource; + @Nullable private final String principal; + @Nullable private final String access; + @Nullable private final String pageToken; + @Nullable private final Integer maxResults; + + public ListPermissionsRequest( + ResourceType resourceType, + @Nullable String database, + @Nullable String table, + @Nullable String function, + @Nullable String view, + @Nullable String principal, + @Nullable String access, + @Nullable String pageToken, + @Nullable Integer maxResults) { + this.resource = exactResource(resourceType, database, table, function, view); + if (!isBlank(principal)) { + PermissionAssignment.validatePrincipal(principal); + } + checkArgument(maxResults == null || maxResults > 0, "maxResults must be greater than 0."); + checkArgument( + maxResults == null || maxResults <= MAX_PAGE_SIZE, + "maxResults must be at most %s.", + MAX_PAGE_SIZE); + this.principal = isBlank(principal) ? null : principal; + this.access = isBlank(access) ? null : PermissionAccess.canonicalize(resource, access); + this.pageToken = isBlank(pageToken) ? null : pageToken; + this.maxResults = maxResults; + } + + public ResourceType getResourceType() { + return resource.getType(); + } + + @Nullable + public String getDatabase() { + return resource.getDatabase(); + } + + @Nullable + public String getTable() { + return resource.getTable(); + } + + @Nullable + public String getFunction() { + return resource.getFunction(); + } + + @Nullable + public String getView() { + return resource.getView(); + } + + @Nullable + public String getPrincipal() { + return principal; + } + + @Nullable + public String getAccess() { + return access; + } + + @Nullable + public String getPageToken() { + return pageToken; + } + + @Nullable + public Integer getMaxResults() { + return maxResults; + } + + public PermissionResource resource() { + return resource; + } + + public ListPermissionsRequest withPageToken(@Nullable String newPageToken) { + return new ListPermissionsRequest( + resource.getType(), + resource.getDatabase(), + resource.getTable(), + resource.getFunction(), + resource.getView(), + principal, + access, + newPageToken, + maxResults); + } + + private static PermissionResource exactResource( + ResourceType resourceType, + @Nullable String database, + @Nullable String table, + @Nullable String function, + @Nullable String view) { + checkNotNull(resourceType, "resourceType cannot be null"); + try { + return new PermissionResource(resourceType, database, table, function, view); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + "Permission listing requires an exact target resource: " + e.getMessage(), e); + } + } + + private static boolean isBlank(@Nullable String value) { + return value == null || value.trim().isEmpty(); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/management/ListPoliciesRequest.java b/paimon-api/src/main/java/org/apache/paimon/management/ListPoliciesRequest.java new file mode 100644 index 000000000000..c8be5b39fc21 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/management/ListPoliciesRequest.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.management; + +import org.apache.paimon.annotation.Experimental; + +import javax.annotation.Nullable; + +import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkNotNull; + +/** Filters for listing policies attached to an exact table resource. */ +@Experimental +public class ListPoliciesRequest { + + private final PermissionResource resource; + @Nullable private final PolicyType type; + @Nullable private final String principal; + @Nullable private final String column; + @Nullable private final String pageToken; + @Nullable private final Integer maxResults; + + public ListPoliciesRequest( + PermissionResource resource, + @Nullable PolicyType type, + @Nullable String principal, + @Nullable String column, + @Nullable String pageToken, + @Nullable Integer maxResults) { + this.resource = checkNotNull(resource, "resource cannot be null"); + resource.validatePolicyAttachment(); + if (!isBlank(principal)) { + PermissionAssignment.validatePrincipal(principal); + } + checkArgument(maxResults == null || maxResults > 0, "maxResults must be greater than 0."); + checkArgument( + maxResults == null || maxResults <= ListPermissionsRequest.MAX_PAGE_SIZE, + "maxResults must be at most %s.", + ListPermissionsRequest.MAX_PAGE_SIZE); + this.type = type; + this.principal = isBlank(principal) ? null : principal; + checkArgument( + isBlank(column) || type == PolicyType.COLUMN_MASKING, + "column filter requires type COLUMN_MASKING."); + this.column = isBlank(column) ? null : column; + this.pageToken = isBlank(pageToken) ? null : pageToken; + this.maxResults = maxResults; + } + + public PermissionResource getResource() { + return resource; + } + + @Nullable + public PolicyType getType() { + return type; + } + + @Nullable + public String getPrincipal() { + return principal; + } + + @Nullable + public String getColumn() { + return column; + } + + @Nullable + public String getPageToken() { + return pageToken; + } + + @Nullable + public Integer getMaxResults() { + return maxResults; + } + + public ListPoliciesRequest withPageToken(@Nullable String newPageToken) { + return new ListPoliciesRequest(resource, type, principal, column, newPageToken, maxResults); + } + + private static boolean isBlank(@Nullable String value) { + return value == null || value.trim().isEmpty(); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/management/PermissionAccess.java b/paimon-api/src/main/java/org/apache/paimon/management/PermissionAccess.java new file mode 100644 index 000000000000..742f6c04834b --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/management/PermissionAccess.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.management; + +import org.apache.paimon.annotation.Experimental; + +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumMap; +import java.util.HashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkNotNull; + +/** Built-in access names and validation for permission assignments. */ +@Experimental +public final class PermissionAccess { + + /** Maximum wire length supported by the portable permission storage contract. */ + public static final int MAX_LENGTH = 32; + + public static final String ALL = "ALL"; + public static final String CREATEDATABASE = "CREATEDATABASE"; + public static final String DESCRIBE = "DESCRIBE"; + public static final String ALTER = "ALTER"; + public static final String DROP = "DROP"; + public static final String CREATETABLE = "CREATETABLE"; + public static final String CREATEFUNCTION = "CREATEFUNCTION"; + public static final String CREATEVIEW = "CREATEVIEW"; + public static final String LIST = "LIST"; + public static final String SELECT = "SELECT"; + public static final String UPDATE = "UPDATE"; + public static final String GRANT = "GRANT"; + + private static final Map> BUILT_INS = builtIns(); + + private PermissionAccess() {} + + public static String canonicalize(String access) { + checkArgument(access != null && !access.trim().isEmpty(), "access cannot be empty."); + checkArgument( + access.length() <= MAX_LENGTH, + "access must contain at most %s characters.", + MAX_LENGTH); + String canonical = access.toUpperCase(Locale.ROOT); + checkArgument( + canonical.length() <= MAX_LENGTH, + "access must contain at most %s characters after canonicalization.", + MAX_LENGTH); + if (BUILT_INS.values().stream().anyMatch(values -> values.contains(canonical))) { + return canonical; + } + throw new IllegalArgumentException(String.format("Unknown access '%s'.", canonical)); + } + + public static String canonicalize(PermissionResource resource, String access) { + checkNotNull(resource, "resource cannot be null"); + String canonical = canonicalize(access); + checkArgument( + BUILT_INS.get(resource.getType()).contains(canonical), + "Access '%s' is not valid for %s.", + canonical, + resource.getType()); + return canonical; + } + + public static Set builtIns(ResourceType type) { + return BUILT_INS.get(checkNotNull(type, "resource type cannot be null")); + } + + private static Map> builtIns() { + Map> accesses = new EnumMap<>(ResourceType.class); + accesses.put(ResourceType.CATALOG, values(ALL, ALTER, DROP, GRANT, CREATEDATABASE)); + accesses.put( + ResourceType.CATALOG_ALL, + values( + ALL, + DESCRIBE, + ALTER, + DROP, + GRANT, + CREATETABLE, + CREATEVIEW, + CREATEFUNCTION, + LIST, + SELECT, + UPDATE)); + accesses.put( + ResourceType.DATABASE, + values( + ALL, + DESCRIBE, + ALTER, + DROP, + GRANT, + CREATETABLE, + CREATEVIEW, + CREATEFUNCTION, + LIST)); + accesses.put(ResourceType.DATABASE_ALL, values(ALL, SELECT, UPDATE, ALTER, DROP, GRANT)); + accesses.put(ResourceType.TABLE, values(ALL, SELECT, UPDATE, ALTER, DROP, GRANT)); + accesses.put(ResourceType.COLUMN, values(SELECT)); + accesses.put(ResourceType.VIEW, values(ALL, SELECT, ALTER, DROP, GRANT)); + accesses.put(ResourceType.FUNCTION, values(ALL, SELECT, ALTER, DROP, GRANT)); + return Collections.unmodifiableMap(accesses); + } + + private static Set values(String... accesses) { + return Collections.unmodifiableSet(new HashSet<>(Arrays.asList(accesses))); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/management/PermissionAssignment.java b/paimon-api/src/main/java/org/apache/paimon/management/PermissionAssignment.java new file mode 100644 index 000000000000..4c7b0c6bf57f --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/management/PermissionAssignment.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.management; + +import org.apache.paimon.annotation.Experimental; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.beans.ConstructorProperties; +import java.time.Instant; +import java.time.format.DateTimeParseException; + +import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkNotNull; + +/** + * Direct permission assignment on one exact resource or explicit descendant scope. + * + *

A {@link ResourceType#COLUMN COLUMN} assignment carries one {@link PermissionColumns} value. + * The column range is mutable assignment content rather than identity: granting the same resource, + * access, and principal replaces the complete range. + * + *

{@code expireTime}, when present, is an exclusive authorization upper bound evaluated against + * the server clock. At or after that instant, the assignment must not authorize access, although an + * expired record may remain listable until cleanup. + */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class PermissionAssignment { + + /** Maximum principal identifier length in the portable REST management contract. */ + public static final int MAX_PRINCIPAL_LENGTH = 128; + + private static final String FIELD_RESOURCE = "resource"; + private static final String FIELD_ACCESS = "access"; + private static final String FIELD_PRINCIPAL = "principal"; + private static final String FIELD_COLUMNS = "columns"; + private static final String FIELD_EXPIRE_TIME = "expireTime"; + + @JsonProperty(FIELD_RESOURCE) + private final PermissionResource resource; + + @JsonProperty(FIELD_ACCESS) + private final String access; + + @JsonProperty(FIELD_PRINCIPAL) + private final String principal; + + @Nullable + @JsonProperty(FIELD_COLUMNS) + @JsonInclude(JsonInclude.Include.NON_NULL) + private final PermissionColumns columns; + + @Nullable + @JsonProperty(FIELD_EXPIRE_TIME) + @JsonInclude(JsonInclude.Include.NON_NULL) + private final String expireTime; + + @JsonCreator + @ConstructorProperties({ + FIELD_RESOURCE, + FIELD_ACCESS, + FIELD_PRINCIPAL, + FIELD_COLUMNS, + FIELD_EXPIRE_TIME + }) + public PermissionAssignment( + @JsonProperty(FIELD_RESOURCE) PermissionResource resource, + @JsonProperty(FIELD_ACCESS) String access, + @JsonProperty(FIELD_PRINCIPAL) String principal, + @Nullable @JsonProperty(FIELD_COLUMNS) PermissionColumns columns, + @Nullable @JsonProperty(FIELD_EXPIRE_TIME) String expireTime) { + this.resource = checkNotNull(resource, "resource cannot be null"); + this.access = PermissionAccess.canonicalize(resource, access); + this.principal = validatePrincipal(principal); + checkArgument( + resource.getType() == ResourceType.COLUMN ? columns != null : columns == null, + resource.getType() == ResourceType.COLUMN + ? "columns is required for COLUMN resource." + : "columns is only valid for COLUMN resource."); + this.columns = columns; + if (expireTime != null) { + try { + Instant instant = Instant.parse(expireTime); + checkArgument( + instant.getNano() % 1_000_000 == 0, + "expireTime must have at most millisecond precision."); + } catch (DateTimeParseException e) { + throw new IllegalArgumentException( + "expireTime must be an ISO-8601 UTC instant.", e); + } + } + this.expireTime = expireTime; + } + + public PermissionAssignment( + PermissionResource resource, + String access, + String principal, + @Nullable String expireTime) { + this(resource, access, principal, null, expireTime); + } + + @JsonGetter(FIELD_RESOURCE) + public PermissionResource getResource() { + return resource; + } + + @JsonGetter(FIELD_ACCESS) + public String getAccess() { + return access; + } + + @JsonGetter(FIELD_PRINCIPAL) + public String getPrincipal() { + return principal; + } + + /** Validates and returns an opaque principal identifier. */ + public static String validatePrincipal(String principal) { + checkArgument( + principal != null && !principal.trim().isEmpty(), "principal cannot be empty."); + checkArgument( + principal.length() <= MAX_PRINCIPAL_LENGTH, + "principal must contain at most %s characters.", + MAX_PRINCIPAL_LENGTH); + return principal; + } + + /** Returns the selected or excluded column range for a COLUMN assignment. */ + @Nullable + @JsonGetter(FIELD_COLUMNS) + public PermissionColumns getColumns() { + return columns; + } + + /** Returns the exclusive authorization upper bound, or null for no expiry. */ + @Nullable + @JsonGetter(FIELD_EXPIRE_TIME) + public String getExpireTime() { + return expireTime; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/management/PermissionColumns.java b/paimon-api/src/main/java/org/apache/paimon/management/PermissionColumns.java new file mode 100644 index 000000000000..105d4fb57500 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/management/PermissionColumns.java @@ -0,0 +1,124 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.management; + +import org.apache.paimon.annotation.Experimental; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.beans.ConstructorProperties; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** + * Included or excluded top-level columns of a column-level permission assignment. + * + *

Exactly one list is present. Included names form an allowlist; excluded names form a denylist. + */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class PermissionColumns { + + private static final String FIELD_COLUMN_NAMES = "columnNames"; + private static final String FIELD_EXCLUDED_COLUMN_NAMES = "excludedColumnNames"; + + @Nullable + @JsonProperty(FIELD_COLUMN_NAMES) + @JsonInclude(JsonInclude.Include.NON_NULL) + private final List columnNames; + + @Nullable + @JsonProperty(FIELD_EXCLUDED_COLUMN_NAMES) + @JsonInclude(JsonInclude.Include.NON_NULL) + private final List excludedColumnNames; + + @JsonCreator + @ConstructorProperties({FIELD_COLUMN_NAMES, FIELD_EXCLUDED_COLUMN_NAMES}) + public PermissionColumns( + @Nullable @JsonProperty(FIELD_COLUMN_NAMES) List columnNames, + @Nullable @JsonProperty(FIELD_EXCLUDED_COLUMN_NAMES) List excludedColumnNames) { + checkArgument( + (columnNames == null) != (excludedColumnNames == null), + "columns must contain exactly one of columnNames or excludedColumnNames."); + this.columnNames = immutableNonEmpty(columnNames, FIELD_COLUMN_NAMES); + this.excludedColumnNames = + immutableNonEmpty(excludedColumnNames, FIELD_EXCLUDED_COLUMN_NAMES); + } + + @Nullable + @JsonGetter(FIELD_COLUMN_NAMES) + public List getColumnNames() { + return columnNames; + } + + @Nullable + @JsonGetter(FIELD_EXCLUDED_COLUMN_NAMES) + public List getExcludedColumnNames() { + return excludedColumnNames; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof PermissionColumns)) { + return false; + } + PermissionColumns that = (PermissionColumns) o; + return Objects.equals(columnNames, that.columnNames) + && Objects.equals(excludedColumnNames, that.excludedColumnNames); + } + + @Override + public int hashCode() { + return Objects.hash(columnNames, excludedColumnNames); + } + + @Nullable + private static List immutableNonEmpty( + @Nullable List columns, String fieldName) { + if (columns == null) { + return null; + } + checkArgument(!columns.isEmpty(), "%s cannot be empty.", fieldName); + for (String column : columns) { + checkArgument( + column != null && !column.trim().isEmpty(), + "%s cannot contain an empty column name.", + fieldName); + } + checkArgument( + new HashSet<>(columns).size() == columns.size(), + "%s cannot contain duplicate column names.", + fieldName); + return Collections.unmodifiableList(new ArrayList<>(columns)); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/management/PermissionManagement.java b/paimon-api/src/main/java/org/apache/paimon/management/PermissionManagement.java new file mode 100644 index 000000000000..3eb669c142d8 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/management/PermissionManagement.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.management; + +import org.apache.paimon.PagedList; +import org.apache.paimon.annotation.Experimental; + +/** Control-plane contract for managing permissions on catalog resources. */ +@Experimental +public interface PermissionManagement { + + PagedList listPermissions(ListPermissionsRequest request); + + void grantPermission(PermissionAssignment assignment); + + void revokePermission(PermissionResource resource, String access, String principal); +} diff --git a/paimon-api/src/main/java/org/apache/paimon/management/PermissionResource.java b/paimon-api/src/main/java/org/apache/paimon/management/PermissionResource.java new file mode 100644 index 000000000000..b3d15373b821 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/management/PermissionResource.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.management; + +import org.apache.paimon.annotation.Experimental; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.beans.ConstructorProperties; +import java.util.Objects; + +import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkNotNull; + +/** Structured reference to a resource or explicit descendant scope inside the REST catalog. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class PermissionResource { + + private static final String FIELD_TYPE = "type"; + private static final String FIELD_DATABASE = "database"; + private static final String FIELD_TABLE = "table"; + private static final String FIELD_FUNCTION = "function"; + private static final String FIELD_VIEW = "view"; + + @JsonProperty(FIELD_TYPE) + private final ResourceType type; + + @Nullable + @JsonProperty(FIELD_DATABASE) + @JsonInclude(JsonInclude.Include.NON_NULL) + private final String database; + + @Nullable + @JsonProperty(FIELD_TABLE) + @JsonInclude(JsonInclude.Include.NON_NULL) + private final String table; + + @Nullable + @JsonProperty(FIELD_FUNCTION) + @JsonInclude(JsonInclude.Include.NON_NULL) + private final String function; + + @Nullable + @JsonProperty(FIELD_VIEW) + @JsonInclude(JsonInclude.Include.NON_NULL) + private final String view; + + @JsonCreator + @ConstructorProperties({FIELD_TYPE, FIELD_DATABASE, FIELD_TABLE, FIELD_FUNCTION, FIELD_VIEW}) + public PermissionResource( + @JsonProperty(FIELD_TYPE) String type, + @Nullable @JsonProperty(FIELD_DATABASE) String database, + @Nullable @JsonProperty(FIELD_TABLE) String table, + @Nullable @JsonProperty(FIELD_FUNCTION) String function, + @Nullable @JsonProperty(FIELD_VIEW) String view) { + this(ResourceType.fromString(type), database, table, function, view); + } + + public PermissionResource( + ResourceType type, + @Nullable String database, + @Nullable String table, + @Nullable String function, + @Nullable String view) { + this.type = checkNotNull(type, "resource type cannot be null"); + validate(type, database, table, function, view); + this.database = blankToNull(database); + this.table = blankToNull(table); + this.function = blankToNull(function); + this.view = blankToNull(view); + } + + @JsonGetter(FIELD_TYPE) + public ResourceType getType() { + return type; + } + + @Nullable + @JsonGetter(FIELD_DATABASE) + public String getDatabase() { + return database; + } + + @Nullable + @JsonGetter(FIELD_TABLE) + public String getTable() { + return table; + } + + @Nullable + @JsonGetter(FIELD_FUNCTION) + public String getFunction() { + return function; + } + + @Nullable + @JsonGetter(FIELD_VIEW) + public String getView() { + return view; + } + + /** Validates that this resource can carry a data policy in the current contract. */ + public void validatePolicyAttachment() { + checkArgument( + type == ResourceType.TABLE, + "Policies can currently be attached only to TABLE resources."); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof PermissionResource)) { + return false; + } + PermissionResource that = (PermissionResource) o; + return type == that.type + && Objects.equals(database, that.database) + && Objects.equals(table, that.table) + && Objects.equals(function, that.function) + && Objects.equals(view, that.view); + } + + @Override + public int hashCode() { + return Objects.hash(type, database, table, function, view); + } + + private static void validate( + ResourceType type, + @Nullable String database, + @Nullable String table, + @Nullable String function, + @Nullable String view) { + switch (type) { + case CATALOG: + case CATALOG_ALL: + checkArgument( + isBlank(database) && isBlank(table) && isBlank(function) && isBlank(view), + "%s resource cannot contain object identifiers.", + type); + break; + case DATABASE: + case DATABASE_ALL: + checkArgument(!isBlank(database), "database is required for %s resource.", type); + checkArgument( + isBlank(table) && isBlank(function) && isBlank(view), + "%s resource cannot contain table, function, or view.", + type); + break; + case TABLE: + case COLUMN: + checkArgument(!isBlank(database), "database is required for %s resource.", type); + checkArgument(!isBlank(table), "table is required for %s resource.", type); + checkArgument( + isBlank(function) && isBlank(view), + "%s resource cannot contain function or view.", + type); + break; + case FUNCTION: + checkArgument(!isBlank(database), "database is required for FUNCTION resource."); + checkArgument(!isBlank(function), "function is required for FUNCTION resource."); + checkArgument( + isBlank(table) && isBlank(view), + "FUNCTION resource cannot contain table or view."); + break; + case VIEW: + checkArgument(!isBlank(database), "database is required for VIEW resource."); + checkArgument(!isBlank(view), "view is required for VIEW resource."); + checkArgument( + isBlank(table) && isBlank(function), + "VIEW resource cannot contain table or function."); + break; + default: + throw new IllegalArgumentException("Unsupported resource type " + type); + } + } + + private static boolean isBlank(@Nullable String value) { + return value == null || value.trim().isEmpty(); + } + + @Nullable + private static String blankToNull(@Nullable String value) { + return isBlank(value) ? null : value; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/management/PolicyManagement.java b/paimon-api/src/main/java/org/apache/paimon/management/PolicyManagement.java new file mode 100644 index 000000000000..6b33195b0629 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/management/PolicyManagement.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.management; + +import org.apache.paimon.PagedList; +import org.apache.paimon.annotation.Experimental; + +import javax.annotation.Nullable; + +/** Control-plane contract for row-filter and column-masking policies. */ +@Experimental +public interface PolicyManagement { + + PagedList listPolicies(ListPoliciesRequest request); + + void createPolicy(DataPolicy policy) throws PolicyAlreadyExistException; + + void dropPolicy( + PermissionResource resource, + PolicyType type, + String principal, + @Nullable String column, + boolean ignoreIfNotExists); + + /** Exception for trying to create a policy that already exists. */ + class PolicyAlreadyExistException extends Exception { + + private final DataPolicy policy; + + public PolicyAlreadyExistException(DataPolicy policy) { + this(policy, null); + } + + public PolicyAlreadyExistException(DataPolicy policy, Throwable cause) { + super(message(policy), cause); + this.policy = policy; + } + + public DataPolicy policy() { + return policy; + } + + private static String message(DataPolicy policy) { + String target = policy.type().name(); + if (policy.getColumnMask() != null) { + target += "(" + policy.getColumnMask().getOnColumn() + ")"; + } + PermissionResource resource = policy.getResource(); + return String.format( + "%s policy for principal '%s' already exists on table '%s.%s'.", + target, policy.getPrincipal(), resource.getDatabase(), resource.getTable()); + } + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/management/PolicyType.java b/paimon-api/src/main/java/org/apache/paimon/management/PolicyType.java new file mode 100644 index 000000000000..1312a2632fed --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/management/PolicyType.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.management; + +import org.apache.paimon.annotation.Experimental; + +import javax.annotation.Nullable; + +import java.util.Locale; + +/** Fine-grained data policy types. */ +@Experimental +public enum PolicyType { + ROW_FILTER, + COLUMN_MASKING; + + @Nullable + public static PolicyType fromString(@Nullable String value) { + return value == null ? null : valueOf(value.toUpperCase(Locale.ROOT)); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/management/ResourceType.java b/paimon-api/src/main/java/org/apache/paimon/management/ResourceType.java new file mode 100644 index 000000000000..ef38471ef67b --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/management/ResourceType.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.management; + +import org.apache.paimon.annotation.Experimental; + +import javax.annotation.Nullable; + +import java.util.Locale; + +/** Resource types and explicit descendant scopes supported by permission management. */ +@Experimental +public enum ResourceType { + CATALOG, + CATALOG_ALL, + DATABASE, + DATABASE_ALL, + TABLE, + COLUMN, + VIEW, + FUNCTION; + + @Nullable + public static ResourceType fromString(@Nullable String value) { + return value == null ? null : valueOf(value.toUpperCase(Locale.ROOT)); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/management/RowFilter.java b/paimon-api/src/main/java/org/apache/paimon/management/RowFilter.java new file mode 100644 index 000000000000..4af32042b593 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/management/RowFilter.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.management; + +import org.apache.paimon.annotation.Experimental; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.beans.ConstructorProperties; +import java.nio.charset.StandardCharsets; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Serialized Paimon predicate for a row-filter policy. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class RowFilter { + + public static final int MAX_PREDICATE_BYTES = 60 * 1024; + + private static final String FIELD_PREDICATE = "predicate"; + + @JsonProperty(FIELD_PREDICATE) + private final String predicate; + + @JsonCreator + @ConstructorProperties({FIELD_PREDICATE}) + public RowFilter(@JsonProperty(FIELD_PREDICATE) String predicate) { + checkArgument(!isBlank(predicate), "predicate cannot be empty."); + checkArgument( + predicate.getBytes(StandardCharsets.UTF_8).length <= MAX_PREDICATE_BYTES, + "predicate must not exceed %s UTF-8 bytes.", + MAX_PREDICATE_BYTES); + this.predicate = predicate; + } + + @JsonGetter(FIELD_PREDICATE) + public String getPredicate() { + return predicate; + } + + private static boolean isBlank(String value) { + return value == null || value.trim().isEmpty(); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java index 238a3969ff04..c4b307de28c9 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTApi.java @@ -20,11 +20,18 @@ import org.apache.paimon.PagedList; import org.apache.paimon.Snapshot; +import org.apache.paimon.annotation.Experimental; import org.apache.paimon.annotation.Public; import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.consumer.ConsumerInfo; import org.apache.paimon.function.FunctionChange; +import org.apache.paimon.management.DataPolicy; +import org.apache.paimon.management.ListPermissionsRequest; +import org.apache.paimon.management.ListPoliciesRequest; +import org.apache.paimon.management.PermissionAssignment; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.PolicyType; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; import org.apache.paimon.partition.PartitionStatistics; @@ -47,14 +54,18 @@ import org.apache.paimon.rest.requests.CreateTagRequest; import org.apache.paimon.rest.requests.CreateViewRequest; import org.apache.paimon.rest.requests.DropPartitionsRequest; +import org.apache.paimon.rest.requests.DropPolicyRequest; import org.apache.paimon.rest.requests.ForwardBranchRequest; +import org.apache.paimon.rest.requests.GrantPermissionRequest; import org.apache.paimon.rest.requests.ListPartitionsByFilterRequest; import org.apache.paimon.rest.requests.ListPartitionsByNamesRequest; import org.apache.paimon.rest.requests.MarkDonePartitionsRequest; +import org.apache.paimon.rest.requests.PolicyRequest; import org.apache.paimon.rest.requests.RegisterTableRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.requests.ReplaceTableRequest; import org.apache.paimon.rest.requests.ResetConsumerRequest; +import org.apache.paimon.rest.requests.RevokePermissionRequest; import org.apache.paimon.rest.requests.RollbackSchemaRequest; import org.apache.paimon.rest.requests.RollbackTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; @@ -79,6 +90,8 @@ import org.apache.paimon.rest.responses.ListFunctionsGloballyResponse; import org.apache.paimon.rest.responses.ListFunctionsResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; +import org.apache.paimon.rest.responses.ListPermissionsResponse; +import org.apache.paimon.rest.responses.ListPoliciesResponse; import org.apache.paimon.rest.responses.ListSnapshotsResponse; import org.apache.paimon.rest.responses.ListTableDetailsResponse; import org.apache.paimon.rest.responses.ListTablesGloballyResponse; @@ -118,6 +131,7 @@ import static org.apache.paimon.rest.RESTUtil.extractPrefixMap; import static org.apache.paimon.rest.auth.AuthProviderFactory.createAuthProvider; import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkNotNull; /** * REST API for REST Catalog. @@ -833,6 +847,97 @@ public AuthTableQueryResponse authTableQuery( restAuthFunction); } + /** Lists permissions on an exact resource in the configured REST catalog. */ + @Experimental + public ListPermissionsResponse listPermissions(ListPermissionsRequest request) { + Map queryParams = Maps.newHashMap(); + putQueryParameter(queryParams, "resourceType", request.getResourceType().name()); + putQueryParameter(queryParams, "database", request.getDatabase()); + putQueryParameter(queryParams, "table", request.getTable()); + putQueryParameter(queryParams, "function", request.getFunction()); + putQueryParameter(queryParams, "view", request.getView()); + putQueryParameter(queryParams, "principal", request.getPrincipal()); + putQueryParameter(queryParams, "access", request.getAccess()); + if (request.getMaxResults() != null) { + queryParams.put(MAX_RESULTS, request.getMaxResults().toString()); + } + putQueryParameter(queryParams, PAGE_TOKEN, request.getPageToken()); + return client.get( + resourcePaths.permissions(), + queryParams, + ListPermissionsResponse.class, + restAuthFunction); + } + + /** Grants a permission for the configured REST catalog. */ + @Experimental + public void grantPermission(PermissionAssignment assignment) { + client.post( + resourcePaths.grantPermission(), + new GrantPermissionRequest(assignment), + restAuthFunction); + } + + /** Revokes a permission for the configured REST catalog. */ + @Experimental + public void revokePermission(PermissionResource resource, String access, String principal) { + client.post( + resourcePaths.revokePermission(), + new RevokePermissionRequest(resource, access, principal), + restAuthFunction); + } + + /** Lists policies attached to an exact table resource. */ + @Experimental + public ListPoliciesResponse listPolicies(ListPoliciesRequest request) { + Map queryParams = Maps.newHashMap(); + if (request.getType() != null) { + putQueryParameter(queryParams, "type", request.getType().name()); + } + putQueryParameter(queryParams, "principal", request.getPrincipal()); + putQueryParameter(queryParams, "column", request.getColumn()); + if (request.getMaxResults() != null) { + queryParams.put(MAX_RESULTS, request.getMaxResults().toString()); + } + putQueryParameter(queryParams, PAGE_TOKEN, request.getPageToken()); + return client.get( + resourcePaths.policies(request.getResource()), + queryParams, + ListPoliciesResponse.class, + restAuthFunction); + } + + /** Creates a principal policy on its attachment resource. */ + @Experimental + public void createPolicy(DataPolicy policy) { + client.post( + resourcePaths.policies(policy.getResource()), + new PolicyRequest(policy), + restAuthFunction); + } + + /** Drops a principal policy from its exact attachment resource. */ + @Experimental + public void dropPolicy( + PermissionResource resource, + PolicyType type, + String principal, + @Nullable String column, + boolean ignoreIfNotExists) { + checkNotNull(resource, "resource cannot be null").validatePolicyAttachment(); + try { + client.delete( + resourcePaths.policies(resource), + new DropPolicyRequest(type, principal, column), + restAuthFunction); + } catch (NoSuchResourceException e) { + if (!ignoreIfNotExists + || !ErrorResponse.RESOURCE_TYPE_POLICY.equals(e.resourceType())) { + throw e; + } + } + } + /** * Drop table. * @@ -1727,4 +1832,11 @@ private final Map buildPagedQueryParams( RESTAuthFunction authFunction() { return restAuthFunction; } + + private static void putQueryParameter( + Map queryParams, String name, @Nullable String value) { + if (StringUtils.isNotEmpty(value)) { + queryParams.put(name, value); + } + } } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTPermissionManagement.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTPermissionManagement.java new file mode 100644 index 000000000000..3091d1ac092a --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTPermissionManagement.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.PagedList; +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.management.ListPermissionsRequest; +import org.apache.paimon.management.PermissionAssignment; +import org.apache.paimon.management.PermissionManagement; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.rest.responses.ListPermissionsResponse; + +/** REST implementation of permission management, bound to the configured REST catalog prefix. */ +@Experimental +public class RESTPermissionManagement implements PermissionManagement { + + private final RESTApi api; + + public RESTPermissionManagement(RESTApi api) { + this.api = api; + } + + @Override + public PagedList listPermissions(ListPermissionsRequest request) { + ListPermissionsResponse response = api.listPermissions(request); + return new PagedList<>(response.getPermissions(), response.getNextPageToken()); + } + + @Override + public void grantPermission(PermissionAssignment assignment) { + api.grantPermission(assignment); + } + + @Override + public void revokePermission(PermissionResource resource, String access, String principal) { + api.revokePermission(resource, access, principal); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/RESTPolicyManagement.java b/paimon-api/src/main/java/org/apache/paimon/rest/RESTPolicyManagement.java new file mode 100644 index 000000000000..45a42f6cf86c --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/RESTPolicyManagement.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.PagedList; +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.management.DataPolicy; +import org.apache.paimon.management.ListPoliciesRequest; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.PolicyManagement; +import org.apache.paimon.management.PolicyManagement.PolicyAlreadyExistException; +import org.apache.paimon.management.PolicyType; +import org.apache.paimon.rest.exceptions.AlreadyExistsException; +import org.apache.paimon.rest.responses.ErrorResponse; +import org.apache.paimon.rest.responses.ListPoliciesResponse; + +import javax.annotation.Nullable; + +/** REST implementation of data policy management for a configured catalog prefix. */ +@Experimental +public class RESTPolicyManagement implements PolicyManagement { + + private final RESTApi api; + + public RESTPolicyManagement(RESTApi api) { + this.api = api; + } + + @Override + public PagedList listPolicies(ListPoliciesRequest request) { + ListPoliciesResponse response = api.listPolicies(request); + return new PagedList<>(response.getPolicies(), response.getNextPageToken()); + } + + @Override + public void createPolicy(DataPolicy policy) throws PolicyAlreadyExistException { + try { + api.createPolicy(policy); + } catch (AlreadyExistsException e) { + if (ErrorResponse.RESOURCE_TYPE_POLICY.equals(e.resourceType())) { + throw new PolicyAlreadyExistException(policy, e); + } + throw e; + } + } + + @Override + public void dropPolicy( + PermissionResource resource, + PolicyType type, + String principal, + @Nullable String column, + boolean ignoreIfNotExists) { + api.dropPolicy(resource, type, principal, column, ignoreIfNotExists); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java index ae09705467cc..5b79d61ac5d3 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/ResourcePaths.java @@ -18,6 +18,8 @@ package org.apache.paimon.rest; +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.management.PermissionResource; import org.apache.paimon.options.Options; import org.apache.paimon.shade.guava30.com.google.common.base.Joiner; @@ -42,6 +44,8 @@ public class ResourcePaths { protected static final String REGISTER = "register"; protected static final String FUNCTIONS = "functions"; protected static final String FUNCTION_DETAILS = "function-details"; + protected static final String PERMISSIONS = "permissions"; + protected static final String POLICIES = "policies"; protected static final String ID = "id"; private static final Joiner SLASH = Joiner.on("/").skipNulls(); @@ -60,6 +64,28 @@ public ResourcePaths(String prefix) { this.prefix = encodeString(prefix); } + @Experimental + public String permissions() { + return SLASH.join(V1, prefix, PERMISSIONS); + } + + @Experimental + public String grantPermission() { + return SLASH.join(permissions(), "grant"); + } + + @Experimental + public String revokePermission() { + return SLASH.join(permissions(), "revoke"); + } + + /** Policy collection nested below its attachment resource. */ + @Experimental + public String policies(PermissionResource resource) { + resource.validatePolicyAttachment(); + return SLASH.join(table(resource.getDatabase(), resource.getTable()), POLICIES); + } + public String databases() { return SLASH.join(V1, prefix, DATABASES); } diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/DropPolicyRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/DropPolicyRequest.java new file mode 100644 index 000000000000..64b7b83c0ef1 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/DropPolicyRequest.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.management.PermissionAssignment; +import org.apache.paimon.management.PolicyType; +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.beans.ConstructorProperties; + +import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkNotNull; + +/** Request for dropping one principal's row filter or column mask. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class DropPolicyRequest implements RESTRequest { + + private static final String FIELD_TYPE = "type"; + private static final String FIELD_PRINCIPAL = "principal"; + private static final String FIELD_COLUMN = "column"; + + private final PolicyType type; + private final String principal; + @Nullable private final String column; + + @JsonCreator + @ConstructorProperties({FIELD_TYPE, FIELD_PRINCIPAL, FIELD_COLUMN}) + public DropPolicyRequest( + @JsonProperty(FIELD_TYPE) PolicyType type, + @JsonProperty(FIELD_PRINCIPAL) String principal, + @Nullable @JsonProperty(FIELD_COLUMN) String column) { + this.type = checkNotNull(type, "policy type cannot be null"); + this.principal = validatePrincipal(principal); + if (type == PolicyType.ROW_FILTER) { + checkArgument(isBlank(column), "ROW_FILTER identity cannot contain a column."); + this.column = null; + } else { + checkArgument(!isBlank(column), "column is required for COLUMN_MASKING identity."); + this.column = column; + } + } + + @JsonGetter(FIELD_TYPE) + public PolicyType getType() { + return type; + } + + @JsonGetter(FIELD_PRINCIPAL) + public String getPrincipal() { + return principal; + } + + @Nullable + @JsonGetter(FIELD_COLUMN) + @JsonInclude(JsonInclude.Include.NON_NULL) + public String getColumn() { + return column; + } + + private static String validatePrincipal(String principal) { + checkArgument( + principal != null && !principal.trim().isEmpty(), "principal cannot be empty."); + checkArgument( + principal.length() <= PermissionAssignment.MAX_PRINCIPAL_LENGTH, + "principal must contain at most %s characters.", + PermissionAssignment.MAX_PRINCIPAL_LENGTH); + return principal; + } + + private static boolean isBlank(@Nullable String value) { + return value == null || value.trim().isEmpty(); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/GrantPermissionRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/GrantPermissionRequest.java new file mode 100644 index 000000000000..377e997adb95 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/GrantPermissionRequest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.management.PermissionAssignment; +import org.apache.paimon.management.PermissionColumns; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.beans.ConstructorProperties; + +/** Request for granting or replacing a permission assignment. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class GrantPermissionRequest implements RESTRequest { + + private static final String FIELD_RESOURCE = "resource"; + private static final String FIELD_ACCESS = "access"; + private static final String FIELD_PRINCIPAL = "principal"; + private static final String FIELD_COLUMNS = "columns"; + private static final String FIELD_EXPIRE_TIME = "expireTime"; + + private final PermissionAssignment assignment; + + public GrantPermissionRequest(PermissionAssignment assignment) { + this.assignment = assignment; + } + + @JsonCreator + @ConstructorProperties({ + FIELD_RESOURCE, + FIELD_ACCESS, + FIELD_PRINCIPAL, + FIELD_COLUMNS, + FIELD_EXPIRE_TIME + }) + public GrantPermissionRequest( + @JsonProperty(FIELD_RESOURCE) PermissionResource resource, + @JsonProperty(FIELD_ACCESS) String access, + @JsonProperty(FIELD_PRINCIPAL) String principal, + @Nullable @JsonProperty(FIELD_COLUMNS) PermissionColumns columns, + @Nullable @JsonProperty(FIELD_EXPIRE_TIME) String expireTime) { + this.assignment = + new PermissionAssignment(resource, access, principal, columns, expireTime); + } + + public PermissionAssignment assignment() { + return assignment; + } + + @JsonGetter(FIELD_RESOURCE) + public PermissionResource getResource() { + return assignment.getResource(); + } + + @JsonGetter(FIELD_ACCESS) + public String getAccess() { + return assignment.getAccess(); + } + + @JsonGetter(FIELD_PRINCIPAL) + public String getPrincipal() { + return assignment.getPrincipal(); + } + + @Nullable + @JsonGetter(FIELD_COLUMNS) + @JsonInclude(JsonInclude.Include.NON_NULL) + public PermissionColumns getColumns() { + return assignment.getColumns(); + } + + @Nullable + @JsonGetter(FIELD_EXPIRE_TIME) + @JsonInclude(JsonInclude.Include.NON_NULL) + public String getExpireTime() { + return assignment.getExpireTime(); + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/PolicyRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/PolicyRequest.java new file mode 100644 index 000000000000..f1e9a0ed53e5 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/PolicyRequest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.management.ColumnMask; +import org.apache.paimon.management.DataPolicy; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.RowFilter; +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnore; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.beans.ConstructorProperties; + +/** Create payload for a principal policy whose table is identified by the request path. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class PolicyRequest implements RESTRequest { + + private static final String FIELD_ROW_FILTER = "rowFilter"; + private static final String FIELD_COLUMN_MASK = "columnMask"; + private static final String FIELD_PRINCIPAL = "principal"; + + @Nullable private final RowFilter rowFilter; + @Nullable private final ColumnMask columnMask; + private final String principal; + + public PolicyRequest(DataPolicy policy) { + this(policy.getRowFilter(), policy.getColumnMask(), policy.getPrincipal()); + } + + @JsonCreator + @ConstructorProperties({FIELD_ROW_FILTER, FIELD_COLUMN_MASK, FIELD_PRINCIPAL}) + public PolicyRequest( + @Nullable @JsonProperty(FIELD_ROW_FILTER) RowFilter rowFilter, + @Nullable @JsonProperty(FIELD_COLUMN_MASK) ColumnMask columnMask, + @JsonProperty(FIELD_PRINCIPAL) String principal) { + this.rowFilter = rowFilter; + this.columnMask = columnMask; + this.principal = principal; + } + + public DataPolicy policy(PermissionResource resource) { + return new DataPolicy(resource, rowFilter, columnMask, principal); + } + + /** Creating a principal policy cannot be replayed after an ambiguous server response. */ + @JsonIgnore + @Override + public boolean isRetrySafe() { + return false; + } + + @Nullable + @JsonGetter(FIELD_ROW_FILTER) + @JsonInclude(JsonInclude.Include.NON_NULL) + public RowFilter getRowFilter() { + return rowFilter; + } + + @Nullable + @JsonGetter(FIELD_COLUMN_MASK) + @JsonInclude(JsonInclude.Include.NON_NULL) + public ColumnMask getColumnMask() { + return columnMask; + } + + @JsonGetter(FIELD_PRINCIPAL) + public String getPrincipal() { + return principal; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/requests/RevokePermissionRequest.java b/paimon-api/src/main/java/org/apache/paimon/rest/requests/RevokePermissionRequest.java new file mode 100644 index 000000000000..0da87420fdcc --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/requests/RevokePermissionRequest.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.requests; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.management.PermissionAccess; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.rest.RESTRequest; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import java.beans.ConstructorProperties; + +/** Idempotent request for revoking a permission assignment by identity. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class RevokePermissionRequest implements RESTRequest { + + private static final String FIELD_RESOURCE = "resource"; + private static final String FIELD_ACCESS = "access"; + private static final String FIELD_PRINCIPAL = "principal"; + + private final PermissionResource resource; + private final String access; + private final String principal; + + @JsonCreator + @ConstructorProperties({FIELD_RESOURCE, FIELD_ACCESS, FIELD_PRINCIPAL}) + public RevokePermissionRequest( + @JsonProperty(FIELD_RESOURCE) PermissionResource resource, + @JsonProperty(FIELD_ACCESS) String access, + @JsonProperty(FIELD_PRINCIPAL) String principal) { + this.resource = resource; + this.access = PermissionAccess.canonicalize(resource, access); + this.principal = + org.apache.paimon.management.PermissionAssignment.validatePrincipal(principal); + } + + @JsonGetter(FIELD_RESOURCE) + public PermissionResource getResource() { + return resource; + } + + @JsonGetter(FIELD_ACCESS) + public String getAccess() { + return access; + } + + @JsonGetter(FIELD_PRINCIPAL) + public String getPrincipal() { + return principal; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java index 4eb52308dd0c..4fe52b1d0b22 100644 --- a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ErrorResponse.java @@ -51,6 +51,8 @@ public class ErrorResponse implements RESTResponse { public static final String RESOURCE_TYPE_FUNCTION = "FUNCTION"; + public static final String RESOURCE_TYPE_POLICY = "POLICY"; + public static final String RESOURCE_TYPE_DEFINITION = "DEFINITION"; private static final String FIELD_MESSAGE = "message"; diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPermissionsResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPermissionsResponse.java new file mode 100644 index 000000000000..ee4d86d1ed9c --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPermissionsResponse.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.management.PermissionAssignment; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.beans.ConstructorProperties; +import java.util.List; + +/** Response for listing permissions. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class ListPermissionsResponse implements PagedResponse { + + private static final String FIELD_PERMISSIONS = "permissions"; + private static final String FIELD_NEXT_PAGE_TOKEN = "nextPageToken"; + + @JsonProperty(FIELD_PERMISSIONS) + private final List permissions; + + @Nullable + @JsonProperty(FIELD_NEXT_PAGE_TOKEN) + @JsonInclude(JsonInclude.Include.NON_NULL) + private final String nextPageToken; + + @JsonCreator + @ConstructorProperties({FIELD_PERMISSIONS, FIELD_NEXT_PAGE_TOKEN}) + public ListPermissionsResponse( + @JsonProperty(FIELD_PERMISSIONS) List permissions, + @Nullable @JsonProperty(FIELD_NEXT_PAGE_TOKEN) String nextPageToken) { + this.permissions = permissions; + this.nextPageToken = nextPageToken; + } + + @JsonGetter(FIELD_PERMISSIONS) + public List getPermissions() { + return permissions; + } + + @Override + @Nullable + @JsonGetter(FIELD_NEXT_PAGE_TOKEN) + public String getNextPageToken() { + return nextPageToken; + } + + @Override + public List data() { + return permissions; + } +} diff --git a/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPoliciesResponse.java b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPoliciesResponse.java new file mode 100644 index 000000000000..95595f066d59 --- /dev/null +++ b/paimon-api/src/main/java/org/apache/paimon/rest/responses/ListPoliciesResponse.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest.responses; + +import org.apache.paimon.annotation.Experimental; +import org.apache.paimon.management.DataPolicy; + +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonCreator; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonGetter; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonInclude; +import org.apache.paimon.shade.jackson2.com.fasterxml.jackson.annotation.JsonProperty; + +import javax.annotation.Nullable; + +import java.beans.ConstructorProperties; +import java.util.List; + +/** Response for listing data policies. */ +@Experimental +@JsonIgnoreProperties(ignoreUnknown = true) +public class ListPoliciesResponse implements PagedResponse { + + private static final String FIELD_POLICIES = "policies"; + private static final String FIELD_NEXT_PAGE_TOKEN = "nextPageToken"; + + private final List policies; + @Nullable private final String nextPageToken; + + @JsonCreator + @ConstructorProperties({FIELD_POLICIES, FIELD_NEXT_PAGE_TOKEN}) + public ListPoliciesResponse( + @JsonProperty(FIELD_POLICIES) List policies, + @Nullable @JsonProperty(FIELD_NEXT_PAGE_TOKEN) String nextPageToken) { + this.policies = policies; + this.nextPageToken = nextPageToken; + } + + @JsonGetter(FIELD_POLICIES) + public List getPolicies() { + return policies; + } + + @Override + @Nullable + @JsonGetter(FIELD_NEXT_PAGE_TOKEN) + @JsonInclude(JsonInclude.Include.NON_NULL) + public String getNextPageToken() { + return nextPageToken; + } + + @Override + public List data() { + return policies; + } +} diff --git a/paimon-api/src/test/java/org/apache/paimon/management/PermissionManagementJsonTest.java b/paimon-api/src/test/java/org/apache/paimon/management/PermissionManagementJsonTest.java new file mode 100644 index 000000000000..839fa67d8f00 --- /dev/null +++ b/paimon-api/src/test/java/org/apache/paimon/management/PermissionManagementJsonTest.java @@ -0,0 +1,558 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.management; + +import org.apache.paimon.rest.RESTApi; +import org.apache.paimon.rest.requests.DropPolicyRequest; +import org.apache.paimon.rest.requests.GrantPermissionRequest; +import org.apache.paimon.rest.requests.PolicyRequest; +import org.apache.paimon.rest.requests.RevokePermissionRequest; + +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** JSON and validation tests for permission and data-policy management contracts. */ +public class PermissionManagementJsonTest { + + private static final String PREDICATE_JSON = + "{\"kind\":\"LEAF\",\"transform\":{\"name\":\"FIELD_REF\"," + + "\"fieldRef\":{\"index\":0,\"name\":\"region\",\"type\":\"STRING\"}}," + + "\"function\":\"EQUAL\",\"literals\":[\"APAC\"]}"; + private static final String TRANSFORM_JSON = + "{\"name\":\"CONCAT\",\"inputs\":[{\"index\":0,\"name\":\"region\"," + + "\"type\":\"STRING\"},\"****\"]}"; + + private static final String ASSIGNMENT_JSON = + "{\"resource\":{\"type\":\"TABLE\",\"database\":\"sales\"," + + "\"table\":\"orders\"},\"access\":\"select\"," + + "\"principal\":\"analyst\"," + + "\"expireTime\":\"2027-01-01T00:00:00Z\"}"; + + private static final String COLUMN_ASSIGNMENT_JSON = + "{\"resource\":{\"type\":\"COLUMN\",\"database\":\"sales\"," + + "\"table\":\"orders\"},\"access\":\"select\"," + + "\"principal\":\"analyst\",\"columns\":{" + + "\"columnNames\":[\"id\",\"region\"]}}"; + + @Test + void testAssignmentDeserializesWithShadedAndExternalJackson() throws Exception { + assertAssignment(RESTApi.fromJson(ASSIGNMENT_JSON, PermissionAssignment.class)); + assertAssignment( + new com.fasterxml.jackson.databind.ObjectMapper() + .readValue(ASSIGNMENT_JSON, PermissionAssignment.class)); + } + + @Test + void testAssignmentDeserializesLowerCaseResourceType() throws Exception { + String lowerCaseJson = ASSIGNMENT_JSON.replace("\"TABLE\"", "\"table\""); + + assertAssignment(RESTApi.fromJson(lowerCaseJson, PermissionAssignment.class)); + assertAssignment( + new com.fasterxml.jackson.databind.ObjectMapper() + .readValue(lowerCaseJson, PermissionAssignment.class)); + } + + @Test + void testColumnAssignmentRoundTripsWithShadedAndExternalJackson() throws Exception { + PermissionAssignment shaded = + RESTApi.fromJson(COLUMN_ASSIGNMENT_JSON, PermissionAssignment.class); + PermissionAssignment external = + new com.fasterxml.jackson.databind.ObjectMapper() + .readValue(COLUMN_ASSIGNMENT_JSON, PermissionAssignment.class); + + for (PermissionAssignment assignment : Arrays.asList(shaded, external)) { + assertThat(assignment.getResource().getType()).isEqualTo(ResourceType.COLUMN); + assertThat(assignment.getAccess()).isEqualTo("SELECT"); + assertThat(assignment.getColumns().getColumnNames()).containsExactly("id", "region"); + assertThat(assignment.getColumns().getExcludedColumnNames()).isNull(); + } + + Map wire = RESTApi.fromJson(RESTApi.toJson(shaded), Map.class); + assertThat(((Map) wire.get("columns")).get("columnNames")) + .isEqualTo(Arrays.asList("id", "region")); + Map externalWire = + new com.fasterxml.jackson.databind.ObjectMapper() + .readValue( + new com.fasterxml.jackson.databind.ObjectMapper() + .writeValueAsString(external), + Map.class); + Map externalColumns = (Map) externalWire.get("columns"); + assertThat(externalColumns.get("columnNames")).isEqualTo(Arrays.asList("id", "region")); + assertThat(externalColumns.get("excludedColumnNames")).isNull(); + } + + @Test + void testGrantAndRevokeUsePrivilegeOnlyWireShapes() throws Exception { + PermissionAssignment assignment = + RESTApi.fromJson(ASSIGNMENT_JSON, PermissionAssignment.class); + Map grant = + RESTApi.fromJson(RESTApi.toJson(new GrantPermissionRequest(assignment)), Map.class); + + assertThat(grant.get("access")).isEqualTo("SELECT"); + assertThat(grant.containsKey("columns")).isFalse(); + assertThat(grant.containsKey("policy")).isFalse(); + assertThat(grant.containsKey("grantOption")).isFalse(); + + Map revoke = + RESTApi.fromJson( + RESTApi.toJson( + new RevokePermissionRequest( + assignment.getResource(), + assignment.getAccess(), + assignment.getPrincipal())), + Map.class); + assertThat(revoke.get("access")).isEqualTo("SELECT"); + assertThat(revoke.containsKey("columns")).isFalse(); + assertThat(revoke.containsKey("policy")).isFalse(); + assertThat(revoke.containsKey("policyType")).isFalse(); + assertThat(revoke.containsKey("grantOption")).isFalse(); + assertThat(revoke.containsKey("expireTime")).isFalse(); + } + + @Test + void testPermissionRequestWithoutExpiry() throws Exception { + String permissionJson = + "{\"resource\":{\"type\":\"TABLE\",\"database\":\"sales\"," + + "\"table\":\"orders\"},\"access\":\"select\"," + + "\"principal\":\"analyst\"}"; + + assertThat(RESTApi.fromJson(permissionJson, GrantPermissionRequest.class).getExpireTime()) + .isNull(); + assertThat(RESTApi.fromJson(permissionJson, RevokePermissionRequest.class).getAccess()) + .isEqualTo("SELECT"); + + String policyJson = + RESTApi.toJson( + new PolicyRequest( + DataPolicy.rowFilter( + tableResource(), + new RowFilter(PREDICATE_JSON), + "analyst"))); + DataPolicy policy = + RESTApi.fromJson(policyJson, PolicyRequest.class).policy(tableResource()); + assertThat(policy.getRowFilter().getPredicate()).isEqualTo(PREDICATE_JSON); + } + + @Test + void testPoliciesArePrincipalScopedPaimonDefinitions() throws Exception { + String json = + RESTApi.toJson( + DataPolicy.columnMask( + tableResource(), + new ColumnMask("email", TRANSFORM_JSON), + "analysts")); + + DataPolicy policy = RESTApi.fromJson(json, DataPolicy.class); + DataPolicy externalPolicy = + new com.fasterxml.jackson.databind.ObjectMapper().readValue(json, DataPolicy.class); + assertThat(policy.type()).isEqualTo(PolicyType.COLUMN_MASKING); + assertThat(policy.getRowFilter()).isNull(); + assertThat(policy.getColumnMask().getOnColumn()).isEqualTo("email"); + assertThat(policy.getColumnMask().getTransform()).isEqualTo(TRANSFORM_JSON); + assertThat(policy.getPrincipal()).isEqualTo("analysts"); + assertThat(externalPolicy.getColumnMask().getTransform()).isEqualTo(TRANSFORM_JSON); + + PolicyRequest policyRequest = new PolicyRequest(policy); + assertThat(policyRequest.isRetrySafe()).isFalse(); + Map request = RESTApi.fromJson(RESTApi.toJson(policyRequest), Map.class); + assertThat(request.containsKey("resource")).isFalse(); + assertThat(request.get("principal")).isEqualTo("analysts"); + assertThat(request.containsKey("type")).isFalse(); + assertThat(request.get("columnMask")).isInstanceOf(Map.class); + Map external = + new com.fasterxml.jackson.databind.ObjectMapper() + .readValue( + new com.fasterxml.jackson.databind.ObjectMapper() + .writeValueAsString(policy), + Map.class); + assertThat(external.containsKey("type")).isFalse(); + assertThat(new ArrayList(((Map) external.get("columnMask")).keySet())) + .containsExactlyInAnyOrder("onColumn", "transform"); + } + + @Test + void testAccessAndPolicyValidation() { + assertThat( + new PermissionAssignment( + catalogResource(), "createdatabase", "analyst", null) + .getAccess()) + .isEqualTo("CREATEDATABASE"); + assertThat( + new PermissionAssignment(databaseResource(), "createview", "analyst", null) + .getAccess()) + .isEqualTo("CREATEVIEW"); + assertThat( + new PermissionAssignment(functionResource(), "select", "analyst", null) + .getAccess()) + .isEqualTo("SELECT"); + assertThat(PermissionAccess.builtIns(ResourceType.CATALOG)) + .containsExactlyInAnyOrder("ALL", "ALTER", "DROP", "GRANT", "CREATEDATABASE"); + assertThat(PermissionAccess.builtIns(ResourceType.CATALOG_ALL)) + .containsExactlyInAnyOrder( + "ALL", + "DESCRIBE", + "ALTER", + "DROP", + "GRANT", + "CREATETABLE", + "CREATEVIEW", + "CREATEFUNCTION", + "LIST", + "SELECT", + "UPDATE"); + assertThat(PermissionAccess.builtIns(ResourceType.DATABASE)) + .containsExactlyInAnyOrder( + "ALL", + "DESCRIBE", + "ALTER", + "DROP", + "GRANT", + "CREATETABLE", + "CREATEVIEW", + "CREATEFUNCTION", + "LIST"); + assertThat(PermissionAccess.builtIns(ResourceType.DATABASE_ALL)) + .containsExactlyInAnyOrder("ALL", "ALTER", "DROP", "SELECT", "UPDATE", "GRANT"); + assertThat(PermissionAccess.builtIns(ResourceType.TABLE)) + .containsExactlyInAnyOrder("ALL", "ALTER", "DROP", "SELECT", "UPDATE", "GRANT"); + assertThat(PermissionAccess.builtIns(ResourceType.VIEW)) + .containsExactlyInAnyOrder("ALL", "ALTER", "DROP", "SELECT", "GRANT"); + assertThat(PermissionAccess.builtIns(ResourceType.FUNCTION)) + .containsExactlyInAnyOrder("ALL", "ALTER", "DROP", "SELECT", "GRANT"); + assertThat(PermissionAccess.builtIns(ResourceType.COLUMN)).containsExactly("SELECT"); + + PermissionColumns included = new PermissionColumns(Arrays.asList("id", "region"), null); + assertThat( + new PermissionAssignment( + columnResource(), "select", "analyst", included, null) + .getColumns()) + .isEqualTo(included); + + assertThatThrownBy( + () -> + new PermissionAssignment( + catalogResource(), "SELECT", "analyst", null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("not valid for CATALOG"); + assertThatThrownBy( + () -> + new PermissionAssignment( + databaseResource(), "SELECT", "analyst", null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("not valid for DATABASE"); + assertThatThrownBy( + () -> + new PermissionAssignment( + catalogAllResource(), "CREATEDATABASE", "analyst", null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("not valid for CATALOG_ALL"); + assertThatThrownBy( + () -> + new PermissionAssignment( + databaseAllResource(), "LIST", "analyst", null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("not valid for DATABASE_ALL"); + assertThatThrownBy( + () -> + new PermissionAssignment( + tableResource(), "CREATEVIEW", "analyst", null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("not valid for TABLE"); + assertThatThrownBy( + () -> + new PermissionAssignment( + functionResource(), "UPDATE", "analyst", null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("not valid for FUNCTION"); + assertThatThrownBy( + () -> + new PermissionAssignment( + columnResource(), "UPDATE", "analyst", included, null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("not valid for COLUMN"); + assertThatThrownBy( + () -> + new PermissionAssignment( + columnResource(), "SELECT", "analyst", null, null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("columns is required"); + assertThatThrownBy( + () -> + new PermissionAssignment( + tableResource(), "SELECT", "analyst", included, null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("only valid for COLUMN"); + assertThatThrownBy( + () -> + new PermissionColumns( + Collections.singletonList("id"), + Collections.singletonList("region"))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("exactly one"); + assertThatThrownBy(() -> new PermissionColumns(Collections.emptyList(), null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot be empty"); + assertThatThrownBy(() -> new PermissionColumns(Arrays.asList("id", "id"), null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("duplicate"); + assertThatThrownBy(() -> new PermissionColumns(Collections.singletonList(" "), null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("empty"); + for (String access : + Arrays.asList( + "USE_CATALOG", + "CREATE_DATABASE", + "USE_DATABASE", + "CREATE_TABLE", + "CREATE_VIEW", + "CREATE_FUNCTION", + "INSERT", + "DELETE", + "EXECUTE", + "MANAGE_PERMISSIONS", + "vendor.example/read_sensitive")) { + assertThatThrownBy( + () -> + new PermissionAssignment( + tableResource(), access, "analyst", null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("Unknown access"); + } + assertThatThrownBy( + () -> + new PermissionAssignment( + tableResource(), + repeat('A', PermissionAccess.MAX_LENGTH + 1), + "analyst", + null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("32"); + assertThatThrownBy( + () -> + new PermissionAssignment( + tableResource(), "a/" + repeat('ß', 16), "analyst", null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("after canonicalization"); + assertThatThrownBy( + () -> + new PermissionAssignment( + tableResource(), + "SELECT", + repeat('p', PermissionAssignment.MAX_PRINCIPAL_LENGTH + 1), + null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("128"); + assertThatThrownBy(() -> new RowFilter(" ")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("predicate"); + assertThatThrownBy(() -> new ColumnMask("email", " ")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("transform"); + assertThatThrownBy( + () -> + new DataPolicy( + tableResource(), + new RowFilter(PREDICATE_JSON), + new ColumnMask("region", TRANSFORM_JSON), + "analyst")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("exactly one"); + assertThatThrownBy( + () -> + new DataPolicy( + catalogResource(), + null, + new ColumnMask("email", TRANSFORM_JSON), + "analyst")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("TABLE"); + assertThatThrownBy( + () -> + new PermissionAssignment( + tableResource(), + "SELECT", + "analyst", + "2027-01-01T00:00:00.000001Z")) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("millisecond"); + } + + @Test + void testPermissionColumnsDefensivelyCopiesItsRange() { + java.util.List source = new ArrayList<>(Arrays.asList("id", "region")); + PermissionColumns columns = new PermissionColumns(source, null); + + source.clear(); + assertThat(columns.getColumnNames()).containsExactly("id", "region"); + assertThatThrownBy(() -> columns.getColumnNames().add("email")) + .isInstanceOf(UnsupportedOperationException.class); + } + + @Test + void testPermissionListRequiresExactResourceAndBoundsPageSize() { + assertThatThrownBy( + () -> + new ListPermissionsRequest( + ResourceType.TABLE, + "sales", + null, + null, + null, + null, + null, + null, + 25)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("exact target"); + assertThatThrownBy( + () -> + new ListPermissionsRequest( + ResourceType.CATALOG, + null, + null, + null, + null, + null, + null, + null, + 1001)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("at most 1000"); + + ListPermissionsRequest databaseAccess = + new ListPermissionsRequest( + ResourceType.DATABASE, + "sales", + null, + null, + null, + null, + "createview", + null, + 25); + assertThat(databaseAccess.getAccess()).isEqualTo("CREATEVIEW"); + assertThatThrownBy( + () -> + new ListPermissionsRequest( + ResourceType.DATABASE, + "sales", + null, + null, + null, + null, + "SELECT", + null, + 25)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("not valid for DATABASE"); + } + + @Test + void testPolicyDefinitionsBoundUtf8PayloadSize() { + assertThatThrownBy(() -> new RowFilter(repeat('p', RowFilter.MAX_PREDICATE_BYTES + 1))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("UTF-8 bytes"); + assertThatThrownBy( + () -> + new ColumnMask( + "email", repeat('t', ColumnMask.MAX_TRANSFORM_BYTES + 1))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("UTF-8 bytes"); + } + + @Test + void testDropPolicyRequestRoundTrip() throws Exception { + DropPolicyRequest request = + new DropPolicyRequest(PolicyType.COLUMN_MASKING, "analyst", "email"); + DropPolicyRequest roundTrip = + RESTApi.fromJson(RESTApi.toJson(request), DropPolicyRequest.class); + + assertThat(roundTrip.getType()).isEqualTo(PolicyType.COLUMN_MASKING); + assertThat(roundTrip.getPrincipal()).isEqualTo("analyst"); + assertThat(roundTrip.getColumn()).isEqualTo("email"); + assertThatThrownBy(() -> new DropPolicyRequest(PolicyType.COLUMN_MASKING, "analyst", null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("column is required"); + } + + @Test + void testResourceCanonicalizesBlankIrrelevantLocators() throws Exception { + PermissionResource catalog = + new PermissionResource(ResourceType.CATALOG, "", " ", null, null); + PermissionResource catalogAll = + new PermissionResource(ResourceType.CATALOG_ALL, "", " ", null, null); + + assertThat(catalog).isEqualTo(catalogResource()); + assertThat(RESTApi.toJson(catalog)).isEqualTo("{\"type\":\"CATALOG\"}"); + assertThat(catalogAll).isEqualTo(catalogAllResource()); + assertThat(RESTApi.toJson(catalogAll)).isEqualTo("{\"type\":\"CATALOG_ALL\"}"); + assertThat(RESTApi.toJson(databaseAllResource())) + .isEqualTo("{\"type\":\"DATABASE_ALL\",\"database\":\"sales\"}"); + } + + private static PermissionResource catalogResource() { + return new PermissionResource(ResourceType.CATALOG, null, null, null, null); + } + + private static PermissionResource catalogAllResource() { + return new PermissionResource(ResourceType.CATALOG_ALL, null, null, null, null); + } + + private static PermissionResource databaseResource() { + return new PermissionResource(ResourceType.DATABASE, "sales", null, null, null); + } + + private static PermissionResource databaseAllResource() { + return new PermissionResource(ResourceType.DATABASE_ALL, "sales", null, null, null); + } + + private static PermissionResource tableResource() { + return new PermissionResource(ResourceType.TABLE, "sales", "orders", null, null); + } + + private static PermissionResource columnResource() { + return new PermissionResource(ResourceType.COLUMN, "sales", "orders", null, null); + } + + private static PermissionResource functionResource() { + return new PermissionResource(ResourceType.FUNCTION, "sales", null, "calculate_tax", null); + } + + private static String repeat(char value, int length) { + char[] values = new char[length]; + Arrays.fill(values, value); + return new String(values); + } + + private static void assertAssignment(PermissionAssignment assignment) { + assertThat(assignment.getResource().getType()).isEqualTo(ResourceType.TABLE); + assertThat(assignment.getResource().getDatabase()).isEqualTo("sales"); + assertThat(assignment.getResource().getTable()).isEqualTo("orders"); + assertThat(assignment.getAccess()).isEqualTo("SELECT"); + assertThat(assignment.getPrincipal()).isEqualTo("analyst"); + assertThat(assignment.getExpireTime()).isEqualTo("2027-01-01T00:00:00Z"); + } +} diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTPermissionManagementTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTPermissionManagementTest.java new file mode 100644 index 000000000000..48d19ffa76a6 --- /dev/null +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTPermissionManagementTest.java @@ -0,0 +1,251 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.PagedList; +import org.apache.paimon.management.ListPermissionsRequest; +import org.apache.paimon.management.PermissionAssignment; +import org.apache.paimon.management.PermissionColumns; +import org.apache.paimon.management.PermissionManagement; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.ResourceType; +import org.apache.paimon.options.Options; +import org.apache.paimon.rest.exceptions.ForbiddenException; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.URLDecoder; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.apache.paimon.rest.RESTCatalogInternalOptions.PREFIX; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN_PROVIDER; +import static org.apache.paimon.rest.RESTCatalogOptions.URI; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Behavioral tests for REST permission management. */ +public class RESTPermissionManagementTest { + + private static final String BASE_PATH = "/v1/catalog+id/permissions"; + + private HttpServer server; + private PermissionManagement management; + private final AtomicReference grantBody = new AtomicReference<>(); + private final AtomicReference revokeBody = new AtomicReference<>(); + private final AtomicReference authorization = new AtomicReference<>(); + private final AtomicReference listQuery = new AtomicReference<>(); + private final AtomicInteger revokeCalls = new AtomicInteger(); + + @BeforeEach + void setUp() throws Exception { + server = HttpServer.create(new InetSocketAddress(0), 0); + server.createContext( + "/v1/", + exchange -> { + authorization.set(exchange.getRequestHeaders().getFirst("Authorization")); + String path = exchange.getRequestURI().getRawPath(); + if (BASE_PATH.equals(path) && "GET".equals(exchange.getRequestMethod())) { + listQuery.set(exchange.getRequestURI().getRawQuery()); + respond( + exchange, + 200, + "{\"permissions\":[{\"resource\":{\"type\":\"TABLE\"," + + "\"database\":\"sales\",\"table\":\"orders\"}," + + "\"access\":\"SELECT\"," + + "\"principal\":\"analyst\"}]," + + "\"nextPageToken\":\"next\"}"); + } else if ((BASE_PATH + "/grant").equals(path)) { + String body = readBody(exchange); + grantBody.set(body); + if (body.contains("denied")) { + respond(exchange, 403, "{\"message\":\"forbidden\",\"code\":403}"); + } else { + respond(exchange, 200, null); + } + } else if ((BASE_PATH + "/revoke").equals(path)) { + revokeBody.set(readBody(exchange)); + revokeCalls.incrementAndGet(); + respond(exchange, 200, null); + } else { + respond(exchange, 404, "{\"message\":\"missing\",\"code\":404}"); + } + }); + server.start(); + + Options options = new Options(); + options.set(URI, "http://127.0.0.1:" + server.getAddress().getPort()); + options.set(TOKEN_PROVIDER, "bear"); + options.set(TOKEN, "secret"); + options.set(PREFIX, "catalog id"); + management = new RESTPermissionManagement(new RESTApi(options, false)); + } + + @AfterEach + void tearDown() { + if (server != null) { + server.stop(0); + } + } + + @Test + void testListUsesEncodedPrefixAndCompleteFilters() throws Exception { + PagedList page = + management.listPermissions( + new ListPermissionsRequest( + ResourceType.TABLE, + "sales", + "orders", + null, + null, + "analyst", + null, + "start", + 25)); + + assertThat(page.getElements()).hasSize(1); + assertThat(page.getElements().get(0).getPrincipal()).isEqualTo("analyst"); + assertThat(page.getNextPageToken()).isEqualTo("next"); + assertThat(queryParameters(listQuery.get())) + .containsEntry("principal", "analyst") + .containsEntry("resourceType", "TABLE") + .containsEntry("database", "sales") + .containsEntry("table", "orders") + .containsEntry("maxResults", "25") + .containsEntry("pageToken", "start"); + assertThat(authorization.get()).isEqualTo("Bearer secret"); + } + + @Test + void testGrantAndRevokeUseStructuredWireShapes() throws Exception { + PermissionAssignment assignment = assignment("analyst"); + management.grantPermission(assignment); + management.revokePermission(assignment.getResource(), "select", assignment.getPrincipal()); + + Map grant = RESTApi.fromJson(grantBody.get(), Map.class); + Map grantResource = (Map) grant.get("resource"); + assertThat(grantResource.get("type")).isEqualTo("TABLE"); + assertThat(grantResource.get("database")).isEqualTo("sales"); + assertThat(grantResource.get("table")).isEqualTo("orders"); + assertThat(grant.get("principal")).isEqualTo("analyst"); + assertThat(grant.containsKey("columns")).isFalse(); + assertThat(grant.containsKey("policy")).isFalse(); + assertThat(grant.containsKey("grantOption")).isFalse(); + assertThat(grant.containsKey("catalog")).isFalse(); + + Map revoke = RESTApi.fromJson(revokeBody.get(), Map.class); + Map revokeResource = (Map) revoke.get("resource"); + assertThat(revokeResource.get("type")).isEqualTo("TABLE"); + assertThat(revokeResource.get("database")).isEqualTo("sales"); + assertThat(revokeResource.get("table")).isEqualTo("orders"); + assertThat(revoke.get("access")).isEqualTo("SELECT"); + assertThat(revoke.get("principal")).isEqualTo("analyst"); + assertThat(revoke.containsKey("expireTime")).isFalse(); + assertThat(revoke.containsKey("grantOption")).isFalse(); + } + + @Test + void testForbiddenGrantPreservesRESTErrorTranslation() { + assertThatThrownBy(() -> management.grantPermission(assignment("denied"))) + .isInstanceOf(ForbiddenException.class) + .hasMessageContaining("forbidden"); + } + + @Test + void testColumnGrantCarriesRangeButRevokeUsesOnlyIdentity() throws Exception { + PermissionAssignment assignment = + new PermissionAssignment( + new PermissionResource(ResourceType.COLUMN, "sales", "orders", null, null), + "SELECT", + "analyst", + new PermissionColumns(Arrays.asList("id", "region"), null), + null); + + management.grantPermission(assignment); + Map grant = RESTApi.fromJson(grantBody.get(), Map.class); + assertThat(((Map) grant.get("columns")).get("columnNames")) + .isEqualTo(Arrays.asList("id", "region")); + + management.revokePermission( + assignment.getResource(), assignment.getAccess(), assignment.getPrincipal()); + Map revoke = RESTApi.fromJson(revokeBody.get(), Map.class); + assertThat(((Map) revoke.get("resource")).get("type")).isEqualTo("COLUMN"); + assertThat(revoke.containsKey("columns")).isFalse(); + } + + @Test + void testRepeatedRevokeIsIdempotent() { + PermissionAssignment assignment = assignment("missing"); + management.revokePermission( + assignment.getResource(), assignment.getAccess(), assignment.getPrincipal()); + management.revokePermission( + assignment.getResource(), assignment.getAccess(), assignment.getPrincipal()); + + assertThat(revokeCalls).hasValue(2); + } + + private static PermissionAssignment assignment(String principal) { + return new PermissionAssignment( + new PermissionResource(ResourceType.TABLE, "sales", "orders", null, null), + "SELECT", + principal, + null); + } + + private static Map queryParameters(String query) throws Exception { + Map values = new HashMap<>(); + for (String parameter : query.split("&")) { + String[] pair = parameter.split("=", 2); + values.put(URLDecoder.decode(pair[0], "UTF-8"), URLDecoder.decode(pair[1], "UTF-8")); + } + return values; + } + + private static String readBody(HttpExchange exchange) throws IOException { + byte[] data = new byte[8192]; + int read = exchange.getRequestBody().read(data); + return read < 0 ? "" : new String(data, 0, read, StandardCharsets.UTF_8); + } + + private static void respond(HttpExchange exchange, int code, String body) throws IOException { + if (body == null) { + exchange.sendResponseHeaders(code, 0); + exchange.getResponseBody().close(); + } else { + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(code, bytes.length); + try (OutputStream output = exchange.getResponseBody()) { + output.write(bytes); + } + exchange.close(); + } + } +} diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/RESTPolicyManagementTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/RESTPolicyManagementTest.java new file mode 100644 index 000000000000..d08b7f404029 --- /dev/null +++ b/paimon-api/src/test/java/org/apache/paimon/rest/RESTPolicyManagementTest.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.PagedList; +import org.apache.paimon.management.ColumnMask; +import org.apache.paimon.management.DataPolicy; +import org.apache.paimon.management.ListPoliciesRequest; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.PolicyManagement; +import org.apache.paimon.management.PolicyManagement.PolicyAlreadyExistException; +import org.apache.paimon.management.PolicyType; +import org.apache.paimon.management.ResourceType; +import org.apache.paimon.options.Options; +import org.apache.paimon.rest.exceptions.AlreadyExistsException; +import org.apache.paimon.rest.exceptions.NoSuchResourceException; +import org.apache.paimon.utils.JsonSerdeUtil; + +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static org.apache.paimon.rest.RESTCatalogInternalOptions.PREFIX; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN; +import static org.apache.paimon.rest.RESTCatalogOptions.TOKEN_PROVIDER; +import static org.apache.paimon.rest.RESTCatalogOptions.URI; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Behavioral tests for REST data-policy management. */ +public class RESTPolicyManagementTest { + + private static final String COLLECTION_PATH = + "/v1/catalog+id/databases/sales/tables/orders/policies"; + + private HttpServer server; + private PolicyManagement management; + private final AtomicReference listQuery = new AtomicReference<>(); + private final AtomicReference createBody = new AtomicReference<>(); + private final AtomicReference createError = new AtomicReference<>(); + private final AtomicReference deleteBody = new AtomicReference<>(); + private final AtomicReference deleteError = new AtomicReference<>(); + private final AtomicInteger deleteCalls = new AtomicInteger(); + + @BeforeEach + void setUp() throws Exception { + server = HttpServer.create(new InetSocketAddress(0), 0); + server.createContext( + "/v1/", + exchange -> { + String path = exchange.getRequestURI().getRawPath(); + String method = exchange.getRequestMethod(); + if (COLLECTION_PATH.equals(path) && "GET".equals(method)) { + listQuery.set(exchange.getRequestURI().getRawQuery()); + respond(exchange, 200, listResponse()); + } else if (COLLECTION_PATH.equals(path) && "POST".equals(method)) { + createBody.set(readBody(exchange)); + if (createError.get() == null) { + respond(exchange, 200, null); + } else { + respond(exchange, 409, createError.get()); + } + } else if (COLLECTION_PATH.equals(path) && "DELETE".equals(method)) { + deleteBody.set(readBody(exchange)); + deleteCalls.incrementAndGet(); + if (deleteError.get() == null) { + respond(exchange, 200, null); + } else { + respond(exchange, 404, deleteError.get()); + } + } else { + respond(exchange, 404, "{\"message\":\"missing\",\"code\":404}"); + } + }); + server.start(); + + Options options = new Options(); + options.set(URI, "http://127.0.0.1:" + server.getAddress().getPort()); + options.set(TOKEN_PROVIDER, "bear"); + options.set(TOKEN, "secret"); + options.set(PREFIX, "catalog id"); + management = new RESTPolicyManagement(new RESTApi(options, false)); + } + + @AfterEach + void tearDown() { + if (server != null) { + server.stop(0); + } + } + + @Test + void testListUsesResourceNestedPathAndIdentityFilters() { + PagedList policies = + management.listPolicies( + new ListPoliciesRequest( + tableResource(), + PolicyType.COLUMN_MASKING, + "analyst", + "email", + "start", + 25)); + + assertThat(policies.getElements()).hasSize(1); + assertThat(policies.getNextPageToken()).isEqualTo("next"); + assertThat(listQuery.get()) + .contains("type=COLUMN_MASKING") + .contains("principal=analyst") + .contains("column=email") + .contains("maxResults=25") + .contains("pageToken=start"); + } + + @Test + void testCreateAndDropUseCrudMethods() throws Exception { + DataPolicy policy = policy(); + management.createPolicy(policy); + management.dropPolicy( + policy.getResource(), + policy.type(), + policy.getPrincipal(), + policy.getColumnMask().getOnColumn(), + false); + + assertThat(createBody.get()).contains("\"principal\":\"analyst\""); + assertThat(createBody.get()).doesNotContain("\"resource\""); + assertThat(deleteBody.get()) + .contains("\"type\":\"COLUMN_MASKING\"") + .contains("\"column\":\"email\""); + assertThat(deleteCalls).hasValue(1); + } + + @Test + void testCreateMapsOnlyPolicyConflict() { + DataPolicy policy = policy(); + createError.set( + "{\"resourceType\":\"POLICY\",\"resourceName\":" + + "\"COLUMN_MASKING:analyst:email\"," + + "\"message\":\"already exists\",\"code\":409}"); + + assertThatThrownBy(() -> management.createPolicy(policy)) + .isInstanceOf(PolicyAlreadyExistException.class) + .hasMessageContaining("COLUMN_MASKING(email)") + .hasCauseInstanceOf(AlreadyExistsException.class); + + createError.set( + "{\"resourceType\":\"TABLE\",\"resourceName\":\"orders\"," + + "\"message\":\"table conflict\",\"code\":409}"); + assertThatThrownBy(() -> management.createPolicy(policy)) + .isInstanceOf(AlreadyExistsException.class) + .hasMessageContaining("table conflict"); + } + + @Test + void testDropIfExistsOnlyIgnoresMissingPolicy() { + DataPolicy policy = policy(); + deleteError.set( + "{\"resourceType\":\"POLICY\",\"resourceName\":" + + "\"COLUMN_MASKING:analyst:email\"," + + "\"message\":\"missing\",\"code\":404}"); + + management.dropPolicy( + policy.getResource(), + policy.type(), + policy.getPrincipal(), + policy.getColumnMask().getOnColumn(), + true); + + deleteError.set( + "{\"resourceType\":\"TABLE\",\"resourceName\":\"orders\"," + + "\"message\":\"missing table\",\"code\":404}"); + assertThatThrownBy( + () -> + management.dropPolicy( + policy.getResource(), + policy.type(), + policy.getPrincipal(), + policy.getColumnMask().getOnColumn(), + true)) + .isInstanceOf(NoSuchResourceException.class) + .hasMessageContaining("missing table"); + } + + private static DataPolicy policy() { + return DataPolicy.columnMask( + tableResource(), + new ColumnMask( + "email", + "{\"name\":\"FIELD_REF\",\"fieldRef\":{\"index\":0," + + "\"name\":\"region\",\"type\":\"STRING\"}}"), + "analyst"); + } + + private static PermissionResource tableResource() { + return new PermissionResource(ResourceType.TABLE, "sales", "orders", null, null); + } + + private static String listResponse() { + return "{\"policies\":[" + policyJson() + "],\"nextPageToken\":\"next\"}"; + } + + private static String policyJson() { + return JsonSerdeUtil.toFlatJson(policy()); + } + + private static String readBody(HttpExchange exchange) throws IOException { + byte[] data = new byte[8192]; + int read = exchange.getRequestBody().read(data); + return read < 0 ? "" : new String(data, 0, read, StandardCharsets.UTF_8); + } + + private static void respond(HttpExchange exchange, int code, String body) throws IOException { + if (body == null) { + exchange.sendResponseHeaders(code, 0); + exchange.getResponseBody().close(); + } else { + byte[] bytes = body.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(code, bytes.length); + try (OutputStream output = exchange.getResponseBody()) { + output.write(bytes); + } + exchange.close(); + } + } +} diff --git a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java index 51bf7c8831f5..9e645db3b79f 100644 --- a/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java +++ b/paimon-api/src/test/java/org/apache/paimon/rest/requests/RequestJacksonCompatibilityTest.java @@ -169,9 +169,13 @@ public class RequestJacksonCompatibilityTest { CreatePartitionsRequest.class, CreateTableRequest.class, CreateViewRequest.class, + DropPolicyRequest.class, + GrantPermissionRequest.class, + PolicyRequest.class, RegisterTableRequest.class, RenameTableRequest.class, ReplaceTableRequest.class, + RevokePermissionRequest.class, RollbackTableRequest.class) .collect(Collectors.toSet()); diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java b/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java index 0524eea1b07c..6d381078c739 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/TableQueryAuthResult.java @@ -48,6 +48,8 @@ import java.util.TreeMap; import java.util.stream.Collectors; +import static org.apache.paimon.utils.Preconditions.checkArgument; + /** Auth result for table query, including row level filter and optional column masking rules. */ public class TableQueryAuthResult implements Serializable { @@ -89,13 +91,10 @@ public Predicate extractPredicate() { if (filter != null && !filter.isEmpty()) { List predicates = new ArrayList<>(); for (String json : filter) { - if (StringUtils.isEmpty(json)) { - continue; - } + checkArgument(!StringUtils.isEmpty(json), "Row filter cannot be empty."); Predicate predicate = JsonSerdeUtil.fromJson(json, Predicate.class); - if (predicate != null) { - predicates.add(predicate); - } + checkArgument(predicate != null, "Row filter cannot be JSON null."); + predicates.add(predicate); } if (predicates.size() == 1) { rowFilter = predicates.get(0); @@ -122,13 +121,10 @@ public Map extractColumnMasking() { for (Map.Entry e : columnMasking.entrySet()) { String column = e.getKey(); String json = e.getValue(); - if (StringUtils.isEmpty(column) || StringUtils.isEmpty(json)) { - continue; - } + checkArgument(!StringUtils.isEmpty(column), "Column mask target cannot be empty."); + checkArgument(!StringUtils.isEmpty(json), "Column mask transform cannot be empty."); Transform transform = JsonSerdeUtil.fromJson(json, Transform.class); - if (transform == null) { - continue; - } + checkArgument(transform != null, "Column mask transform cannot be JSON null."); result.put(column, transform); } } @@ -184,14 +180,15 @@ private static Map transformRemapping( for (Map.Entry e : masking.entrySet()) { String targetColumn = e.getKey(); Transform transform = e.getValue(); - if (targetColumn == null || transform == null) { - continue; - } + checkArgument(targetColumn != null, "Column mask target cannot be null."); + checkArgument(transform != null, "Column mask transform cannot be null."); int targetIndex = outputRowType.getFieldIndex(targetColumn); - if (targetIndex < 0) { - continue; - } + checkArgument( + targetIndex >= 0, + "Column mask target '%s' is not present in output row type %s.", + targetColumn, + outputRowType); List newInputs = new ArrayList<>(); for (Object input : transform.inputs()) { @@ -234,7 +231,7 @@ public Predicate visit(LeafPredicate predicate) { String fieldName = ref.name(); int newIndex = outputRowType.getFieldIndex(fieldName); if (newIndex < 0) { - throw new RuntimeException( + throw new IllegalArgumentException( String.format( "Unable to read data without column %s when row filter enabled.", fieldName)); @@ -250,15 +247,20 @@ public Predicate visit(LeafPredicate predicate) { @Override public Predicate visit(CompoundPredicate predicate) { + checkArgument( + predicate.function() != null, "Compound row filter function cannot be null."); + checkArgument( + predicate.children() != null, "Compound row filter children cannot be null."); List remappedChildren = new ArrayList<>(); for (Predicate child : predicate.children()) { + checkArgument(child != null, "Compound row filter child cannot be null."); Predicate remapped = child.visit(this); if (remapped != null) { remappedChildren.add(remapped); } } if (remappedChildren.isEmpty()) { - return null; + throw new IllegalArgumentException("Compound row filter must contain a predicate."); } if (remappedChildren.size() == 1) { return remappedChildren.get(0); diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index a24fb2354912..9a21f3fa41e2 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -22,6 +22,7 @@ import org.apache.paimon.PagedList; import org.apache.paimon.Snapshot; import org.apache.paimon.TableType; +import org.apache.paimon.annotation.Experimental; import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.CatalogContext; @@ -39,6 +40,8 @@ import org.apache.paimon.fs.cache.LocalCacheManager; import org.apache.paimon.function.Function; import org.apache.paimon.function.FunctionChange; +import org.apache.paimon.management.PermissionManagement; +import org.apache.paimon.management.PolicyManagement; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; import org.apache.paimon.partition.PartitionStatistics; @@ -137,6 +140,16 @@ public RESTCatalogLoader catalogLoader() { return new RESTCatalogLoader(context); } + @Experimental + public PermissionManagement permissionManagement() { + return new RESTPermissionManagement(api); + } + + @Experimental + public PolicyManagement policyManagement() { + return new RESTPolicyManagement(api); + } + @Override public List listDatabases() { return api.listDatabases(); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java index 59e8cc0666ee..6d6f572d2b12 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/source/AbstractDataTableRead.java @@ -21,8 +21,10 @@ import org.apache.paimon.catalog.TableQueryAuthResult; import org.apache.paimon.data.InternalRow; import org.apache.paimon.disk.IOManager; +import org.apache.paimon.predicate.FieldRef; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateProjectionConverter; +import org.apache.paimon.predicate.Transform; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.types.RowType; @@ -35,6 +37,7 @@ import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; @@ -138,9 +141,21 @@ private RecordReader authedReader(Split split, TableQueryAuthResult RowType tableType = schema.logicalRowType(); RowType readType = this.readType == null ? tableType : this.readType; Predicate authPredicate = authResult.extractPredicate(); + Map columnMasking = authResult.extractColumnMasking(); ProjectedRow backRow = null; + Set authFields = new HashSet<>(); if (authPredicate != null) { - Set authFields = collectFieldNames(authPredicate); + authFields.addAll(collectFieldNames(authPredicate)); + } + for (Map.Entry mask : columnMasking.entrySet()) { + authFields.add(mask.getKey()); + for (Object input : mask.getValue().inputs()) { + if (input instanceof FieldRef) { + authFields.add(((FieldRef) input).name()); + } + } + } + if (!authFields.isEmpty()) { List readFields = readType.getFieldNames(); List authAddNames = new ArrayList<>(); Set readFieldSet = new HashSet<>(readFields); @@ -151,7 +166,7 @@ private RecordReader authedReader(Split split, TableQueryAuthResult } if (!authAddNames.isEmpty()) { readType = tableType.project(ListUtils.union(readFields, authAddNames)); - withReadType(readType); + applyReadType(readType); backRow = ProjectedRow.from(readType.projectIndexes(readFields)); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java new file mode 100644 index 000000000000..3d45924871f1 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/TableQueryAuthResultTest.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.catalog; + +import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.JsonSerdeUtil; + +import org.junit.jupiter.api.Test; + +import java.util.Collections; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +/** Tests that malformed query-authorization definitions cannot be silently ignored. */ +public class TableQueryAuthResultTest { + + @Test + void testInvalidRowFilterFailsClosed() { + assertThatThrownBy( + () -> + new TableQueryAuthResult(Collections.singletonList(""), null) + .extractPredicate()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot be empty"); + assertThatThrownBy( + () -> + new TableQueryAuthResult(Collections.singletonList("null"), null) + .extractPredicate()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("JSON null"); + + Predicate emptyCompound = + JsonSerdeUtil.fromJson( + "{\"kind\":\"COMPOUND\",\"function\":\"AND\",\"children\":[]}", + Predicate.class); + assertThatThrownBy(() -> TableQueryAuthResult.remapPredicate(emptyCompound, RowType.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("must contain a predicate"); + + Predicate missingFunction = + JsonSerdeUtil.fromJson( + "{\"kind\":\"COMPOUND\",\"function\":null,\"children\":[" + + "{\"kind\":\"LEAF\",\"transform\":{\"name\":\"NULL\"}," + + "\"function\":\"TRUE\",\"literals\":[]}," + + "{\"kind\":\"LEAF\",\"transform\":{\"name\":\"NULL\"}," + + "\"function\":\"TRUE\",\"literals\":[]}]}", + Predicate.class); + assertThatThrownBy(() -> TableQueryAuthResult.remapPredicate(missingFunction, RowType.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("function cannot be null"); + } + + @Test + void testInvalidColumnMaskFailsClosed() { + assertThatThrownBy( + () -> + new TableQueryAuthResult( + null, Collections.singletonMap("email", "")) + .extractColumnMasking()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot be empty"); + assertThatThrownBy( + () -> + new TableQueryAuthResult( + null, Collections.singletonMap("email", "null")) + .extractColumnMasking()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("JSON null"); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKey.java b/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKey.java new file mode 100644 index 000000000000..325c1bceb2ad --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKey.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.management.DataPolicy; +import org.apache.paimon.management.PolicyType; + +import javax.annotation.Nullable; + +import java.util.Objects; + +/** Stable identity of a policy stored by the REST catalog test server. */ +final class PolicyKey implements Comparable { + + final String tableUuid; + final PolicyType type; + final String principal; + @Nullable final String column; + + PolicyKey(String tableUuid, DataPolicy policy) { + this( + tableUuid, + policy.type(), + policy.getPrincipal(), + policy.getColumnMask() == null ? null : policy.getColumnMask().getOnColumn()); + } + + PolicyKey(String tableUuid, PolicyType type, String principal, @Nullable String column) { + this.tableUuid = tableUuid; + this.type = type; + this.principal = principal; + this.column = column; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof PolicyKey)) { + return false; + } + PolicyKey that = (PolicyKey) o; + return tableUuid.equals(that.tableUuid) + && type == that.type + && principal.equals(that.principal) + && Objects.equals(column, that.column); + } + + @Override + public int hashCode() { + return Objects.hash(tableUuid, type, principal, column); + } + + @Override + public int compareTo(PolicyKey that) { + int result = tableUuid.compareTo(that.tableUuid); + if (result != 0) { + return result; + } + result = type.compareTo(that.type); + if (result != 0) { + return result; + } + result = principal.compareTo(that.principal); + if (result != 0) { + return result; + } + if (column == null) { + return that.column == null ? 0 : -1; + } + return that.column == null ? 1 : column.compareTo(that.column); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKeyTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKeyTest.java new file mode 100644 index 000000000000..027736ded674 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/PolicyKeyTest.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.management.ColumnMask; +import org.apache.paimon.management.DataPolicy; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.ResourceType; + +import org.junit.jupiter.api.Test; + +import java.util.TreeSet; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for policy identities used by the REST catalog test server. */ +public class PolicyKeyTest { + + @Test + void testOrderingDoesNotFlattenOpaquePrincipalAndColumn() { + PermissionResource resource = + new PermissionResource(ResourceType.TABLE, "database", "table", null, null); + PolicyKey first = + new PolicyKey( + "table-id", + DataPolicy.columnMask( + resource, new ColumnMask("c", "{\"name\":\"NULL\"}"), "a:b")); + PolicyKey second = + new PolicyKey( + "table-id", + DataPolicy.columnMask( + resource, new ColumnMask("b:c", "{\"name\":\"NULL\"}"), "a")); + + TreeSet sorted = new TreeSet<>(); + sorted.add(first); + sorted.add(second); + + assertThat(sorted).containsExactly(second, first); + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index 733b4b1c9fef..8a59aaa2a268 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -40,11 +40,19 @@ import org.apache.paimon.function.FunctionChange; import org.apache.paimon.function.FunctionDefinition; import org.apache.paimon.function.FunctionImpl; +import org.apache.paimon.management.ColumnMask; +import org.apache.paimon.management.DataPolicy; +import org.apache.paimon.management.ListPermissionsRequest; +import org.apache.paimon.management.PermissionAssignment; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.ResourceType; +import org.apache.paimon.management.RowFilter; import org.apache.paimon.operation.Lock; import org.apache.paimon.options.Options; import org.apache.paimon.partition.Partition; import org.apache.paimon.partition.PartitionStatistics; import org.apache.paimon.partition.PartitionUtils; +import org.apache.paimon.predicate.FieldRef; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.Transform; import org.apache.paimon.rest.auth.AuthProvider; @@ -63,12 +71,16 @@ import org.apache.paimon.rest.requests.CreateTagRequest; import org.apache.paimon.rest.requests.CreateViewRequest; import org.apache.paimon.rest.requests.DropPartitionsRequest; +import org.apache.paimon.rest.requests.DropPolicyRequest; +import org.apache.paimon.rest.requests.GrantPermissionRequest; import org.apache.paimon.rest.requests.ListPartitionsByFilterRequest; import org.apache.paimon.rest.requests.ListPartitionsByNamesRequest; import org.apache.paimon.rest.requests.MarkDonePartitionsRequest; +import org.apache.paimon.rest.requests.PolicyRequest; import org.apache.paimon.rest.requests.RenameTableRequest; import org.apache.paimon.rest.requests.ReplaceTableRequest; import org.apache.paimon.rest.requests.ResetConsumerRequest; +import org.apache.paimon.rest.requests.RevokePermissionRequest; import org.apache.paimon.rest.requests.RollbackSchemaRequest; import org.apache.paimon.rest.requests.RollbackTableRequest; import org.apache.paimon.rest.responses.AlterDatabaseResponse; @@ -93,6 +105,8 @@ import org.apache.paimon.rest.responses.ListFunctionsGloballyResponse; import org.apache.paimon.rest.responses.ListFunctionsResponse; import org.apache.paimon.rest.responses.ListPartitionsResponse; +import org.apache.paimon.rest.responses.ListPermissionsResponse; +import org.apache.paimon.rest.responses.ListPoliciesResponse; import org.apache.paimon.rest.responses.ListSnapshotsResponse; import org.apache.paimon.rest.responses.ListTableDetailsResponse; import org.apache.paimon.rest.responses.ListTablesGloballyResponse; @@ -161,6 +175,7 @@ import java.util.Queue; import java.util.Set; import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -185,6 +200,7 @@ import static org.apache.paimon.rest.ResourcePaths.TABLE_DETAILS; import static org.apache.paimon.rest.ResourcePaths.VIEWS; import static org.apache.paimon.rest.ResourcePaths.VIEW_DETAILS; +import static org.apache.paimon.utils.Preconditions.checkArgument; /** Mock REST server for testing. */ public class RESTCatalogServer { @@ -195,13 +211,22 @@ public class RESTCatalogServer { public static final String AUTHORIZATION_HEADER_KEY = "Authorization"; private final String databaseUri; + private final String permissionUri; private final CatalogContext catalogContext; private final RESTFileSystemCatalog catalog; private final MockWebServer server; private final Map databaseStore = new HashMap<>(); - private final Map tableMetadataStore = new HashMap<>(); + private final Map tableMetadataStore = new ConcurrentHashMap<>(); + private final RESTPermissionStore permissionStore = new RESTPermissionStore(); + private final Map policyStore = new ConcurrentHashMap<>(); + private final Map tablePolicyLocks = new ConcurrentHashMap<>(); + private final TableLifecycleLocks tableLifecycleLocks = new TableLifecycleLocks(); + private final Set managementPrincipals = new HashSet<>(); + private final Set queryPrincipals = new HashSet<>(); + private final Set noManagementPermissionResources = + ConcurrentHashMap.newKeySet(); private final List receivedListPartitionsByFilterRequests = new java.util.concurrent.CopyOnWriteArrayList<>(); @@ -238,6 +263,7 @@ public RESTCatalogServer( this.configResponse.getDefaults().get(RESTCatalogInternalOptions.PREFIX.key()); this.resourcePaths = new ResourcePaths(prefix); this.databaseUri = resourcePaths.databases(); + this.permissionUri = resourcePaths.permissions(); Options conf = new Options(); this.configResponse.getDefaults().forEach(conf::setString); conf.setString(WAREHOUSE.key(), dataPath); @@ -332,11 +358,36 @@ public void addTableColumnAuth(Identifier identifier, List select) { } public void setRowFilterAuth(Identifier identifier, List rowFilters) { - rowFilterAuthHandler.put(identifier.getFullName(), rowFilters); + if (rowFilters == null) { + rowFilterAuthHandler.remove(identifier.getFullName()); + } else { + rowFilterAuthHandler.put(identifier.getFullName(), rowFilters); + } } public void setColumnMaskingAuth(Identifier identifier, Map columnMasking) { - columnMaskingAuthHandler.put(identifier.getFullName(), columnMasking); + if (columnMasking == null) { + columnMaskingAuthHandler.remove(identifier.getFullName()); + } else { + columnMaskingAuthHandler.put(identifier.getFullName(), columnMasking); + } + } + + public void registerManagementPrincipal(String principal) { + managementPrincipals.add(principal); + } + + public void setQueryPrincipals(Set principals) { + queryPrincipals.clear(); + queryPrincipals.addAll(principals); + } + + public void denyManagementPermission(PermissionResource resource) { + noManagementPermissionResources.add(resource); + } + + public void allowManagementPermission(PermissionResource resource) { + noManagementPermissionResources.remove(resource); } public RESTToken getDataToken(Identifier identifier) { @@ -384,6 +435,13 @@ public MockResponse dispatch(RecordedRequest request) { .queryParameter(WAREHOUSE.key()) .equals(warehouse)) { return mockResponse(configResponse, 200); + } else if (permissionUri.equals(resourcePath) + || request.getPath().startsWith(permissionUri + "/")) { + return permissionsApiHandler( + request.getMethod(), resourcePath, data, parameters); + } else if (isPolicyPath(resourcePath)) { + return policiesApiHandler( + request.getMethod(), resourcePath, data, parameters); } else if (databaseUri.equals(request.getPath()) || request.getPath().contains(databaseUri + "?")) { return databasesApiHandler(restAuthParameter.method(), data, parameters); @@ -996,6 +1054,17 @@ private MockResponse authTable(Identifier identifier, String data) throws Except if (metadata == null) { throw new Catalog.TableNotExistException(identifier); } + synchronized (policyLock(metadata.uuid())) { + TableMetadata current = tableMetadataStore.get(identifier.getFullName()); + if (current == null || !current.uuid().equals(metadata.uuid())) { + throw new Catalog.TableNotExistException(identifier); + } + return authTable(identifier, requestBody, current); + } + } + + private MockResponse authTable( + Identifier identifier, AuthTableQueryRequest requestBody, TableMetadata metadata) { List columnAuth = columnAuthHandler.get(identifier.getFullName()); if (columnAuth != null) { List select = requestBody.select(); @@ -1009,20 +1078,61 @@ private MockResponse authTable(Identifier identifier, String data) throws Except } }); } - List rowFilters = rowFilterAuthHandler.get(identifier.getFullName()); + if (!RESTColumnPermissionSupport.canSelect( + permissionStore, queryPrincipals, identifier, metadata, requestBody.select())) { + throw new Catalog.TableNoPermissionException(identifier); + } + List rowFilters = + new ArrayList<>( + rowFilterAuthHandler.getOrDefault( + identifier.getFullName(), Collections.emptyList())); Map columnMasking = - columnMaskingAuthHandler.get(identifier.getFullName()); + new HashMap<>( + columnMaskingAuthHandler.getOrDefault( + identifier.getFullName(), Collections.emptyMap())); + for (Map.Entry entry : policyStore.entrySet()) { + if (!entry.getKey().tableUuid.equals(metadata.uuid()) + || !appliesToQueryPrincipal(entry.getValue())) { + continue; + } + DataPolicy policy = entry.getValue(); + RowFilter rowFilter = policy.getRowFilter(); + if (rowFilter != null) { + Predicate predicate; + try { + predicate = parseRowFilter(metadata.schema(), rowFilter); + } catch (RuntimeException e) { + throw new Catalog.TableNoPermissionException(identifier); + } + if (predicate == null) { + throw new Catalog.TableNoPermissionException(identifier); + } + rowFilters.add(predicate); + continue; + } + ColumnMask columnMask = policy.getColumnMask(); + Transform transform; + try { + transform = parseColumnMask(metadata.schema(), columnMask); + } catch (RuntimeException e) { + throw new Catalog.TableNoPermissionException(identifier); + } + if (transform == null || columnMasking.containsKey(columnMask.getOnColumn())) { + throw new Catalog.TableNoPermissionException(identifier); + } + columnMasking.put(columnMask.getOnColumn(), transform); + } // Convert Predicate list to JSON string list List filterJsonList = null; - if (rowFilters != null) { + if (!rowFilters.isEmpty()) { filterJsonList = rowFilters.stream().map(JsonSerdeUtil::toFlatJson).collect(Collectors.toList()); } // Convert Transform map to JSON string map Map columnMaskingJsonMap = null; - if (columnMasking != null) { + if (!columnMasking.isEmpty()) { columnMaskingJsonMap = columnMasking.entrySet().stream() .collect( @@ -1039,6 +1149,10 @@ private MockResponse authTable(Identifier identifier, String data) throws Except return mockResponse(response, 200); } + private boolean appliesToQueryPrincipal(DataPolicy policy) { + return queryPrincipals.contains(policy.getPrincipal()); + } + private MockResponse commitTableHandle(Identifier identifier, String data) throws Exception { CommitTableRequest requestBody = RESTApi.fromJson(data, CommitTableRequest.class); if (noPermissionTables.contains(identifier.getFullName())) { @@ -1496,6 +1610,7 @@ private MockResponse databaseHandle(String method, String data, String databaseN return mockResponse(response, 200); case "DELETE": catalog.dropDatabase(databaseName, false, true); + removeDatabaseTableState(databaseName); databaseStore.remove(databaseName); return new MockResponse().setResponseCode(200); case "POST": @@ -1540,6 +1655,31 @@ private MockResponse databaseHandle(String method, String data, String databaseN return new MockResponse().setResponseCode(404); } + private void removeDatabaseTableState(String databaseName) { + List tableNames = + tableMetadataStore.keySet().stream() + .filter( + tableName -> + databaseName.equals( + Identifier.fromString(tableName).getDatabaseName())) + .collect(Collectors.toList()); + for (String tableName : tableNames) { + synchronized (tableLifecycleLocks.lock(tableName)) { + TableMetadata metadata = tableMetadataStore.get(tableName); + if (metadata == null) { + continue; + } + synchronized (policyLock(metadata.uuid())) { + if (tableMetadataStore.remove(tableName, metadata)) { + removePolicies(metadata.uuid()); + tableLatestSnapshotStore.remove(tableName); + tablePartitionsStore.remove(tableName); + } + } + } + } + } + private MockResponse tablesHandle( String method, String data, String databaseName, Map parameters) throws Exception { @@ -1552,25 +1692,29 @@ private MockResponse tablesHandle( CreateTableRequest requestBody = RESTApi.fromJson(data, CreateTableRequest.class); Identifier identifier = requestBody.getIdentifier(); - Schema schema = requestBody.getSchema(); - TableMetadata tableMetadata; - if (isObjectTable(schema)) { - tableMetadata = createObjectTable(identifier, schema); - } else { - catalog.createTable(identifier, schema, false); - boolean isExternal = - schema.options() != null - && schema.options().containsKey(PATH.key()); - tableMetadata = - createTableMetadata( - requestBody.getIdentifier(), - 0L, - requestBody.getSchema(), - UUID.randomUUID().toString(), - isExternal); + synchronized (tableLifecycleLocks.lock(identifier.getFullName())) { + if (tableMetadataStore.containsKey(identifier.getFullName())) { + throw new Catalog.TableAlreadyExistException(identifier); + } + Schema schema = requestBody.getSchema(); + TableMetadata tableMetadata; + if (isObjectTable(schema)) { + tableMetadata = createObjectTable(identifier, schema); + } else { + catalog.createTable(identifier, schema, false); + boolean isExternal = + schema.options() != null + && schema.options().containsKey(PATH.key()); + tableMetadata = + createTableMetadata( + requestBody.getIdentifier(), + 0L, + requestBody.getSchema(), + UUID.randomUUID().toString(), + isExternal); + } + tableMetadataStore.put(identifier.getFullName(), tableMetadata); } - tableMetadataStore.put( - requestBody.getIdentifier().getFullName(), tableMetadata); return new MockResponse().setResponseCode(200); default: return new MockResponse().setResponseCode(404); @@ -1815,20 +1959,29 @@ private MockResponse tableHandle(String method, String data, Identifier identifi alterTableImpl(identifier, requestBody.getChanges()); return new MockResponse().setResponseCode(200); case "DELETE": - if (!tableMetadataStore.containsKey(identifier.getFullName())) { - return new MockResponse().setResponseCode(404); - } - tableMetadata = tableMetadataStore.get(identifier.getFullName()); - if (!tableMetadata.isExternal()) { - try { - catalog.dropTable(identifier, false); - } catch (Exception e) { - System.out.println(e.getMessage()); + synchronized (tableLifecycleLocks.lock(identifier.getFullName())) { + tableMetadata = tableMetadataStore.get(identifier.getFullName()); + if (tableMetadata == null) { + return new MockResponse().setResponseCode(404); + } + synchronized (policyLock(tableMetadata.uuid())) { + TableMetadata current = tableMetadataStore.get(identifier.getFullName()); + if (current == null || !current.uuid().equals(tableMetadata.uuid())) { + return new MockResponse().setResponseCode(404); + } + if (!current.isExternal()) { + try { + catalog.dropTable(identifier, false); + } catch (Exception e) { + System.out.println(e.getMessage()); + } + } + removePolicies(current.uuid()); + tableMetadataStore.remove(identifier.getFullName(), current); + tableLatestSnapshotStore.remove(identifier.getFullName()); + tablePartitionsStore.remove(identifier.getFullName()); } } - tableMetadataStore.remove(identifier.getFullName()); - tableLatestSnapshotStore.remove(identifier.getFullName()); - tablePartitionsStore.remove(identifier.getFullName()); return new MockResponse().setResponseCode(200); default: return new MockResponse().setResponseCode(404); @@ -1838,32 +1991,49 @@ private MockResponse tableHandle(String method, String data, Identifier identifi private MockResponse replaceTableHandle(Identifier identifier, String data) throws Exception { ReplaceTableRequest requestBody = RESTApi.fromJson(data, ReplaceTableRequest.class); Schema newSchema = requestBody.getSchema(); - if (!tableMetadataStore.containsKey(identifier.getFullName())) { - throw new Catalog.TableNotExistException(identifier); - } TableMetadata tableMetadata = tableMetadataStore.get(identifier.getFullName()); - if (isFormatTable(tableMetadata.schema().toSchema()) || isFormatTable(newSchema)) { - throw new UnsupportedOperationException("replaceTable does not support format tables."); + if (tableMetadata == null) { + throw new Catalog.TableNotExistException(identifier); } - catalog.replaceTable(identifier, newSchema, false); - TableSchema replacedSchema = catalog.loadTableSchema(identifier); - TableMetadata newTableMetadata = - createTableMetadata( - identifier, - replacedSchema.id(), - replacedSchema.toSchema(), - tableMetadata.uuid(), - tableMetadata.isExternal()); - tableMetadataStore.put(identifier.getFullName(), newTableMetadata); - FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); - Snapshot truncateSnapshot = table.snapshotManager().latestSnapshot(); - if (truncateSnapshot != null) { - tableLatestSnapshotStore.put( - identifier.getFullName(), new TableSnapshot(truncateSnapshot, 0L, 0L, 0L, 0L)); - } else { - tableLatestSnapshotStore.remove(identifier.getFullName()); + synchronized (policyLock(tableMetadata.uuid())) { + TableMetadata current = tableMetadataStore.get(identifier.getFullName()); + if (current == null || !current.uuid().equals(tableMetadata.uuid())) { + throw new Catalog.TableNotExistException(identifier); + } + TableSchema replacementSchema = + createTableMetadata( + identifier, + current.schema().id() + 1, + newSchema, + current.uuid(), + current.isExternal()) + .schema(); + validatePoliciesForSchema(identifier, current.uuid(), replacementSchema); + if (isFormatTable(current.schema().toSchema()) || isFormatTable(newSchema)) { + throw new UnsupportedOperationException( + "replaceTable does not support format tables."); + } + catalog.replaceTable(identifier, newSchema, false); + TableSchema replacedSchema = catalog.loadTableSchema(identifier); + TableMetadata newTableMetadata = + createTableMetadata( + identifier, + replacedSchema.id(), + replacedSchema.toSchema(), + current.uuid(), + current.isExternal()); + tableMetadataStore.put(identifier.getFullName(), newTableMetadata); + FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); + Snapshot truncateSnapshot = table.snapshotManager().latestSnapshot(); + if (truncateSnapshot != null) { + tableLatestSnapshotStore.put( + identifier.getFullName(), + new TableSnapshot(truncateSnapshot, 0L, 0L, 0L, 0L)); + } else { + tableLatestSnapshotStore.remove(identifier.getFullName()); + } + tablePartitionsStore.remove(identifier.getFullName()); } - tablePartitionsStore.remove(identifier.getFullName()); return new MockResponse().setResponseCode(200); } @@ -1873,18 +2043,37 @@ private MockResponse renameTableHandle(String data) throws Exception { Identifier toTable = requestBody.getDestination(); if (noPermissionTables.contains(fromTable.getFullName())) { throw new Catalog.TableNoPermissionException(fromTable); - } else if (tableMetadataStore.containsKey(fromTable.getFullName())) { - TableMetadata tableMetadata = tableMetadataStore.get(fromTable.getFullName()); - if (!isFormatTable(tableMetadata.schema().toSchema()) && !tableMetadata.isExternal()) { - catalog.renameTable(requestBody.getSource(), requestBody.getDestination(), false); - } - if (tableMetadataStore.containsKey(toTable.getFullName())) { - throw new Catalog.TableAlreadyExistException(toTable); + } + Object[] locks = + tableLifecycleLocks.ordered(fromTable.getFullName(), toTable.getFullName()); + synchronized (locks[0]) { + synchronized (locks[1]) { + TableMetadata observed = tableMetadataStore.get(fromTable.getFullName()); + if (observed == null) { + throw new Catalog.TableNotExistException(fromTable); + } + synchronized (policyLock(observed.uuid())) { + TableMetadata current = tableMetadataStore.get(fromTable.getFullName()); + if (current == null || !current.uuid().equals(observed.uuid())) { + throw new Catalog.TableNotExistException(fromTable); + } + if (tableMetadataStore.containsKey(toTable.getFullName())) { + throw new Catalog.TableAlreadyExistException(toTable); + } + if (!isFormatTable(current.schema().toSchema()) && !current.isExternal()) { + catalog.renameTable(fromTable, toTable, false); + } + TableMetadata renamedMetadata = + createTableMetadata( + toTable, + current.schema().id(), + current.schema().toSchema(), + current.uuid(), + current.isExternal()); + tableMetadataStore.remove(fromTable.getFullName(), current); + tableMetadataStore.put(toTable.getFullName(), renamedMetadata); + } } - tableMetadataStore.remove(fromTable.getFullName()); - tableMetadataStore.put(toTable.getFullName(), tableMetadata); - } else { - throw new Catalog.TableNotExistException(fromTable); } return new MockResponse().setResponseCode(200); } @@ -2804,45 +2993,52 @@ private MockResponse renameViewHandle(String data) throws Exception { protected void alterTableImpl(Identifier identifier, List changes) throws Catalog.TableNotExistException, Catalog.ColumnAlreadyExistException, Catalog.ColumnNotExistException { - if (tableMetadataStore.containsKey(identifier.getFullName())) { - TableMetadata tableMetadata = tableMetadataStore.get(identifier.getFullName()); - try { - TableSchema schema = tableMetadata.schema(); - if (isFormatTable(schema.toSchema())) { - TableSchema newSchema = + TableMetadata tableMetadata = tableMetadataStore.get(identifier.getFullName()); + if (tableMetadata != null) { + synchronized (policyLock(tableMetadata.uuid())) { + TableMetadata current = tableMetadataStore.get(identifier.getFullName()); + if (current == null || !current.uuid().equals(tableMetadata.uuid())) { + throw new Catalog.TableNotExistException(identifier); + } + try { + TableSchema schema = current.schema(); + TableSchema candidateSchema = SchemaManager.generateTableSchema( schema, changes, new LazyField<>(() -> false), new LazyField<>(() -> identifier)); + validatePoliciesForSchema(identifier, current.uuid(), candidateSchema); + if (isFormatTable(schema.toSchema())) { + TableMetadata newTableMetadata = + createTableMetadata( + identifier, + candidateSchema.id(), + candidateSchema.toSchema(), + current.uuid(), + current.isExternal()); + tableMetadataStore.put(identifier.getFullName(), newTableMetadata); + return; + } + catalog.alterTable(identifier, changes, false); + FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); + TableSchema newSchema = table.schema(); TableMetadata newTableMetadata = createTableMetadata( identifier, newSchema.id(), newSchema.toSchema(), - tableMetadata.uuid(), - tableMetadata.isExternal()); + current.uuid(), + current.isExternal()); tableMetadataStore.put(identifier.getFullName(), newTableMetadata); - return; + } catch (Catalog.TableNotExistException + | Catalog.ColumnAlreadyExistException + | Catalog.ColumnNotExistException + | RuntimeException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException(e); } - catalog.alterTable(identifier, changes, false); - FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); - TableSchema newSchema = table.schema(); - TableMetadata newTableMetadata = - createTableMetadata( - identifier, - newSchema.id(), - newSchema.toSchema(), - tableMetadata.uuid(), - tableMetadata.isExternal()); - tableMetadataStore.put(identifier.getFullName(), newTableMetadata); - } catch (Catalog.TableNotExistException - | Catalog.ColumnAlreadyExistException - | Catalog.ColumnNotExistException - | RuntimeException e) { - throw e; - } catch (Exception e) { - throw new RuntimeException(e); } } } @@ -3113,6 +3309,571 @@ private static int getMaxResults(Map parameters) { return maxResults; } + private MockResponse permissionsApiHandler( + String method, String resourcePath, String data, Map parameters) + throws JsonProcessingException { + if ("GET".equals(method) && permissionUri.equals(resourcePath)) { + PermissionResource target = permissionResource(parameters); + MockResponse authorization = validateManagementPermission(target); + if (authorization != null) { + return authorization; + } + MockResponse validation = validateResourceAndPrincipal(target, parameters); + if (validation != null) { + return validation; + } + List filtered = permissionStore.list(target, parameters); + int start = + parameters.containsKey(PAGE_TOKEN) + ? Integer.parseInt(parameters.get(PAGE_TOKEN)) + : 0; + int end = Math.min(start + getPermissionMaxResults(parameters), filtered.size()); + List page = new ArrayList<>(filtered.subList(start, end)); + String nextPageToken = end < filtered.size() ? String.valueOf(end) : null; + return mockResponse(new ListPermissionsResponse(page, nextPageToken), 200); + } + + if ("POST".equals(method) && (permissionUri + "/grant").equals(resourcePath)) { + PermissionAssignment assignment = + RESTApi.fromJson(data, GrantPermissionRequest.class).assignment(); + MockResponse authorization = validateManagementPermission(assignment.getResource()); + if (authorization != null) { + return authorization; + } + MockResponse validation = + validateResourceAndPrincipal( + assignment.getResource(), assignment.getPrincipal()); + if (validation != null) { + return validation; + } + validation = validateColumnAssignment(assignment); + if (validation != null) { + return validation; + } + permissionStore.put(assignment); + return new MockResponse().setResponseCode(200); + } + + if ("POST".equals(method) && (permissionUri + "/revoke").equals(resourcePath)) { + RevokePermissionRequest request = RESTApi.fromJson(data, RevokePermissionRequest.class); + MockResponse authorization = validateManagementPermission(request.getResource()); + if (authorization != null) { + return authorization; + } + MockResponse validation = + validateResourceAndPrincipal(request.getResource(), request.getPrincipal()); + if (validation != null) { + return validation; + } + permissionStore.remove( + request.getResource(), request.getAccess(), request.getPrincipal()); + return new MockResponse().setResponseCode(200); + } + + return new MockResponse().setResponseCode(404); + } + + @Nullable + private MockResponse validateResourceAndPrincipal( + PermissionResource resource, Map parameters) { + MockResponse resourceError = validateResource(resource); + if (resourceError != null || !parameters.containsKey("principal")) { + return resourceError; + } + return validatePrincipal(parameters.get("principal")); + } + + @Nullable + private MockResponse validateResourceAndPrincipal( + PermissionResource resource, String principal) { + MockResponse resourceError = validateResource(resource); + return resourceError == null ? validatePrincipal(principal) : resourceError; + } + + @Nullable + private MockResponse validateColumnAssignment(PermissionAssignment assignment) { + if (assignment.getResource().getType() != ResourceType.COLUMN) { + return null; + } + PermissionResource resource = assignment.getResource(); + Identifier identifier = Identifier.create(resource.getDatabase(), resource.getTable()); + TableMetadata metadata = tableMetadataStore.get(identifier.getFullName()); + RESTColumnPermissionSupport.ValidationError error = + RESTColumnPermissionSupport.validate(assignment, metadata); + return error == null + ? null + : mockResponse( + new ErrorResponse( + error.resourceType, error.resourceName, error.message, error.code), + error.code); + } + + @Nullable + private MockResponse validateResource(PermissionResource resource) { + boolean exists; + switch (resource.getType()) { + case CATALOG: + case CATALOG_ALL: + exists = true; + break; + case DATABASE: + case DATABASE_ALL: + exists = databaseStore.containsKey(resource.getDatabase()); + break; + case TABLE: + case COLUMN: + exists = + tableMetadataStore.containsKey( + Identifier.create(resource.getDatabase(), resource.getTable()) + .getFullName()); + break; + case FUNCTION: + exists = + functionStore.containsKey( + Identifier.create(resource.getDatabase(), resource.getFunction()) + .getFullName()); + break; + case VIEW: + exists = + viewStore.containsKey( + Identifier.create(resource.getDatabase(), resource.getView()) + .getFullName()); + break; + default: + exists = false; + } + return exists + ? null + : mockResponse( + new ErrorResponse( + resource.getType().name(), + resourceName(resource), + "Permission resource does not exist.", + 404), + 404); + } + + @Nullable + private MockResponse validateManagementPermission(PermissionResource resource) { + return noManagementPermissionResources.contains(resource) + ? mockResponse( + new ErrorResponse( + resource.getType().name(), + resourceName(resource), + "The caller cannot manage permissions on this resource.", + 403), + 403) + : null; + } + + @Nullable + private MockResponse validatePrincipal(String principal) { + return managementPrincipals.contains(principal) + ? null + : mockResponse( + new ErrorResponse( + "PRINCIPAL", + principal, + "Permission principal does not exist.", + 404), + 404); + } + + private static String resourceName(PermissionResource resource) { + switch (resource.getType()) { + case CATALOG: + case CATALOG_ALL: + return "catalog"; + case DATABASE: + case DATABASE_ALL: + return resource.getDatabase(); + case TABLE: + case COLUMN: + return resource.getDatabase() + "." + resource.getTable(); + case FUNCTION: + return resource.getDatabase() + "." + resource.getFunction(); + case VIEW: + return resource.getDatabase() + "." + resource.getView(); + default: + return resource.getType().name(); + } + } + + private static int getPermissionMaxResults(Map parameters) { + String strMaxResults = parameters.get(MAX_RESULTS); + if (strMaxResults == null) { + return DEFAULT_MAX_RESULTS; + } + int maxResults = Integer.parseInt(strMaxResults); + return Math.max(1, Math.min(maxResults, ListPermissionsRequest.MAX_PAGE_SIZE)); + } + + private static boolean matches( + Map parameters, String key, @Nullable String value) { + return !parameters.containsKey(key) || Objects.equals(parameters.get(key), value); + } + + private static PermissionResource permissionResource(Map parameters) { + return new PermissionResource( + ResourceType.fromString(parameters.get("resourceType")), + parameters.get("database"), + parameters.get("table"), + parameters.get("function"), + parameters.get("view")); + } + + private boolean isPolicyPath(String resourcePath) { + try { + policyPath(resourcePath); + return true; + } catch (IllegalArgumentException e) { + return false; + } + } + + private MockResponse policiesApiHandler( + String method, String resourcePath, String data, Map parameters) + throws JsonProcessingException { + PolicyPath path = policyPath(resourcePath); + MockResponse authorization = validateManagementPermission(path.resource); + if (authorization != null) { + return authorization; + } + MockResponse resourceError = validateResource(path.resource); + if (resourceError != null) { + return resourceError; + } + String tableUuid = tableUuid(path.resource); + if ("GET".equals(method)) { + if (parameters.containsKey("principal")) { + MockResponse principalError = validatePrincipal(parameters.get("principal")); + if (principalError != null) { + return principalError; + } + } + List filtered = + policyStore.entrySet().stream() + .filter(entry -> entry.getKey().tableUuid.equals(tableUuid)) + .map(entry -> withResource(entry.getValue(), path.resource)) + .filter(policy -> matchesPolicy(policy, parameters)) + .sorted( + Comparator.comparing( + policy -> new PolicyKey(tableUuid, policy))) + .collect(Collectors.toList()); + int start = + parameters.containsKey(PAGE_TOKEN) + ? Integer.parseInt(parameters.get(PAGE_TOKEN)) + : 0; + int end = Math.min(start + getPermissionMaxResults(parameters), filtered.size()); + String nextPageToken = end < filtered.size() ? String.valueOf(end) : null; + return mockResponse( + new ListPoliciesResponse( + new ArrayList<>(filtered.subList(start, end)), nextPageToken), + 200); + } + + if ("POST".equals(method)) { + DataPolicy policy = RESTApi.fromJson(data, PolicyRequest.class).policy(path.resource); + String resourceName = policyResourceName(policy); + synchronized (policyLock(tableUuid)) { + MockResponse targetError = validatePolicyTableVersion(path.resource, tableUuid); + if (targetError != null) { + return targetError; + } + MockResponse validation = validatePolicy(policy); + if (validation != null) { + return validation; + } + policy = canonicalizePolicy(policy); + PolicyKey key = new PolicyKey(tableUuid, policy); + if (policyStore.putIfAbsent(key, policy) != null) { + return mockResponse( + new ErrorResponse( + ErrorResponse.RESOURCE_TYPE_POLICY, + resourceName, + "Policy already exists.", + 409), + 409); + } + } + return new MockResponse().setResponseCode(200); + } + + if ("DELETE".equals(method)) { + DropPolicyRequest request = RESTApi.fromJson(data, DropPolicyRequest.class); + DataPolicy existing; + synchronized (policyLock(tableUuid)) { + MockResponse targetError = validatePolicyTableVersion(path.resource, tableUuid); + if (targetError != null) { + return targetError; + } + existing = + policyStore.remove( + new PolicyKey( + tableUuid, + request.getType(), + request.getPrincipal(), + request.getColumn())); + } + if (existing == null) { + return mockResponse( + new ErrorResponse( + ErrorResponse.RESOURCE_TYPE_POLICY, + policyResourceName(request), + "Policy does not exist.", + 404), + 404); + } + return new MockResponse().setResponseCode(200); + } + + return new MockResponse().setResponseCode(404); + } + + @Nullable + private MockResponse validatePolicy(DataPolicy policy) { + Identifier identifier = + Identifier.create( + policy.getResource().getDatabase(), policy.getResource().getTable()); + TableMetadata metadata = tableMetadataStore.get(identifier.getFullName()); + if (!CoreOptions.fromMap(metadata.schema().options()).queryAuthEnabled()) { + return mockResponse( + new ErrorResponse( + ErrorResponse.RESOURCE_TYPE_TABLE, + identifier.getFullName(), + "Data policies require the target table option query-auth.enabled=true.", + 409), + 409); + } + MockResponse principalError = validatePrincipal(policy.getPrincipal()); + if (principalError != null) { + return principalError; + } + + RowFilter rowFilter = policy.getRowFilter(); + try { + Set columns = new HashSet<>(metadata.schema().fieldNames()); + ColumnMask columnMask = policy.getColumnMask(); + if (columnMask != null) { + checkArgument( + columns.contains(columnMask.getOnColumn()), + "Policy column %s does not exist in table %s.", + columnMask.getOnColumn(), + identifier.getFullName()); + } + if (rowFilter == null) { + parseColumnMask(metadata.schema(), columnMask); + } else { + parseRowFilter(metadata.schema(), rowFilter); + } + } catch (RuntimeException e) { + return mockResponse( + new ErrorResponse( + ErrorResponse.RESOURCE_TYPE_POLICY, + policyResourceName(policy), + e.getMessage(), + 400), + 400); + } + return null; + } + + private static String policyResourceName(DataPolicy policy) { + ColumnMask columnMask = policy.getColumnMask(); + return policy.type().name() + + ":" + + policy.getPrincipal() + + (columnMask == null ? "" : ":" + columnMask.getOnColumn()); + } + + private static String policyResourceName(DropPolicyRequest request) { + return request.getType().name() + + ":" + + request.getPrincipal() + + (request.getColumn() == null ? "" : ":" + request.getColumn()); + } + + private String tableUuid(PermissionResource resource) { + return tableMetadataStore + .get(Identifier.create(resource.getDatabase(), resource.getTable()).getFullName()) + .uuid(); + } + + private Object policyLock(String tableUuid) { + return tablePolicyLocks.computeIfAbsent(tableUuid, ignored -> new Object()); + } + + @Nullable + private MockResponse validatePolicyTableVersion( + PermissionResource resource, String expectedTableUuid) { + MockResponse resourceError = validateResource(resource); + if (resourceError != null) { + return resourceError; + } + String currentTableUuid = tableUuid(resource); + return expectedTableUuid.equals(currentTableUuid) + ? null + : mockResponse( + new ErrorResponse( + ErrorResponse.RESOURCE_TYPE_TABLE, + resourceName(resource), + "Table changed while managing its policies.", + 409), + 409); + } + + private static DataPolicy withResource(DataPolicy policy, PermissionResource resource) { + return policy.getRowFilter() == null + ? DataPolicy.columnMask(resource, policy.getColumnMask(), policy.getPrincipal()) + : DataPolicy.rowFilter(resource, policy.getRowFilter(), policy.getPrincipal()); + } + + private DataPolicy canonicalizePolicy(DataPolicy policy) { + Identifier identifier = + Identifier.create( + policy.getResource().getDatabase(), policy.getResource().getTable()); + TableSchema schema = tableMetadataStore.get(identifier.getFullName()).schema(); + if (policy.getRowFilter() != null) { + String predicate = + JsonSerdeUtil.toFlatJson(parseRowFilter(schema, policy.getRowFilter())); + return DataPolicy.rowFilter( + policy.getResource(), new RowFilter(predicate), policy.getPrincipal()); + } + ColumnMask columnMask = policy.getColumnMask(); + String transform = JsonSerdeUtil.toFlatJson(parseColumnMask(schema, columnMask)); + return DataPolicy.columnMask( + policy.getResource(), + new ColumnMask(columnMask.getOnColumn(), transform), + policy.getPrincipal()); + } + + private static Predicate parseRowFilter(TableSchema schema, RowFilter rowFilter) { + Predicate predicate = JsonSerdeUtil.fromJson(rowFilter.getPredicate(), Predicate.class); + checkArgument(predicate != null, "Row filter predicate cannot be JSON null."); + Predicate remapped = + TableQueryAuthResult.remapPredicate(predicate, schema.logicalRowType()); + checkArgument(remapped != null, "Row filter predicate cannot be empty."); + return remapped; + } + + private static Transform parseColumnMask(TableSchema schema, ColumnMask columnMask) { + Transform transform = JsonSerdeUtil.fromJson(columnMask.getTransform(), Transform.class); + checkArgument(transform != null, "Column mask transform cannot be JSON null."); + RowType rowType = schema.logicalRowType(); + List remappedInputs = new ArrayList<>(); + for (Object input : transform.inputs()) { + if (input instanceof FieldRef) { + FieldRef ref = (FieldRef) input; + int index = rowType.getFieldIndex(ref.name()); + checkArgument( + index >= 0, + "Column masking refers to field '%s' which is not present in table schema.", + ref.name()); + remappedInputs.add(new FieldRef(index, ref.name(), rowType.getTypeAt(index))); + } else { + remappedInputs.add(input); + } + } + Transform remapped = transform.copyWithNewInputs(remappedInputs); + int targetIndex = rowType.getFieldIndex(columnMask.getOnColumn()); + checkArgument( + targetIndex >= 0, + "Policy column %s does not exist in table schema.", + columnMask.getOnColumn()); + checkArgument( + rowType.getTypeAt(targetIndex).equals(remapped.outputType()), + "Column mask output type %s does not match target column %s type %s.", + remapped.outputType(), + columnMask.getOnColumn(), + rowType.getTypeAt(targetIndex)); + return remapped; + } + + private void removePolicies(@Nullable String tableUuid) { + if (tableUuid != null) { + policyStore.keySet().removeIf(key -> key.tableUuid.equals(tableUuid)); + } + } + + private void validatePoliciesForSchema( + Identifier identifier, @Nullable String tableUuid, TableSchema schema) { + if (tableUuid == null) { + return; + } + List policies = + policyStore.entrySet().stream() + .filter(entry -> entry.getKey().tableUuid.equals(tableUuid)) + .map(Map.Entry::getValue) + .collect(Collectors.toList()); + if (policies.isEmpty()) { + return; + } + checkArgument( + CoreOptions.fromMap(schema.options()).queryAuthEnabled(), + "Cannot disable query-auth.enabled while table %s has data policies.", + identifier.getFullName()); + + Set columns = new HashSet<>(schema.fieldNames()); + for (DataPolicy policy : policies) { + ColumnMask columnMask = policy.getColumnMask(); + if (columnMask != null) { + checkArgument( + columns.contains(columnMask.getOnColumn()), + "Cannot remove or rename policy column %s from table %s.", + columnMask.getOnColumn(), + identifier.getFullName()); + } + if (policy.getRowFilter() == null) { + parseColumnMask(schema, columnMask); + } else { + parseRowFilter(schema, policy.getRowFilter()); + } + } + } + + private static boolean matchesPolicy(DataPolicy policy, Map parameters) { + if (!matches(parameters, "type", policy.type().name())) { + return false; + } + if (parameters.containsKey("column")) { + ColumnMask columnMask = policy.getColumnMask(); + if (columnMask == null + || !Objects.equals(parameters.get("column"), columnMask.getOnColumn())) { + return false; + } + } + return !parameters.containsKey("principal") + || policy.getPrincipal().equals(parameters.get("principal")); + } + + private PolicyPath policyPath(String resourcePath) { + String catalogBase = StringUtils.substringBeforeLast(permissionUri, "/"); + checkArgument(resourcePath.startsWith(catalogBase + "/"), "Not a catalog policy path."); + String[] parts = resourcePath.substring(catalogBase.length() + 1).split("/"); + if (parts.length == 5 + && "databases".equals(parts[0]) + && "tables".equals(parts[2]) + && "policies".equals(parts[4])) { + return new PolicyPath( + new PermissionResource( + ResourceType.TABLE, + RESTUtil.decodeString(parts[1]), + RESTUtil.decodeString(parts[3]), + null, + null)); + } + throw new IllegalArgumentException("Not a policy path."); + } + + private static class PolicyPath { + + private final PermissionResource resource; + + private PolicyPath(PermissionResource resource) { + this.resource = resource; + } + } + private String getNextPageTokenForEntities(List entities, Integer maxResults) { if (entities == null || entities.isEmpty() diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTColumnPermissionSupport.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTColumnPermissionSupport.java new file mode 100644 index 000000000000..39f7d72d3305 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTColumnPermissionSupport.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.catalog.TableMetadata; +import org.apache.paimon.management.PermissionAssignment; +import org.apache.paimon.management.PermissionColumns; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.ResourceType; +import org.apache.paimon.rest.responses.ErrorResponse; + +import javax.annotation.Nullable; + +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** Column permission validation and composition used by the REST catalog test server. */ +final class RESTColumnPermissionSupport { + + private RESTColumnPermissionSupport() {} + + static boolean canSelect( + RESTPermissionStore store, + Set principals, + Identifier identifier, + TableMetadata metadata, + @Nullable List selectedColumns) { + PermissionResource resource = + new PermissionResource( + ResourceType.COLUMN, + identifier.getDatabaseName(), + identifier.getTableName(), + null, + null); + List assignments = + store.list(resource, Collections.emptyMap()).stream() + .filter(assignment -> principals.contains(assignment.getPrincipal())) + .filter(RESTColumnPermissionSupport::notExpired) + .collect(Collectors.toList()); + if (assignments.isEmpty()) { + return true; + } + + Set included = new HashSet<>(metadata.schema().fieldNames()); + for (PermissionAssignment assignment : assignments) { + PermissionColumns columns = assignment.getColumns(); + if (columns.getColumnNames() != null) { + included.retainAll(columns.getColumnNames()); + } else { + included.removeAll(columns.getExcludedColumnNames()); + } + } + List selected = + selectedColumns == null ? metadata.schema().fieldNames() : selectedColumns; + for (String column : selected) { + int nestedSeparator = column.indexOf('.'); + String topLevel = nestedSeparator < 0 ? column : column.substring(0, nestedSeparator); + if (!included.contains(topLevel)) { + return false; + } + } + return true; + } + + @Nullable + static ValidationError validate(PermissionAssignment assignment, TableMetadata metadata) { + if (!CoreOptions.fromMap(metadata.schema().options()).queryAuthEnabled()) { + return new ValidationError( + ErrorResponse.RESOURCE_TYPE_TABLE, + assignment.getResource().getDatabase() + + "." + + assignment.getResource().getTable(), + "Column permissions require query-auth.enabled=true.", + 409); + } + Set tableColumns = new HashSet<>(metadata.schema().fieldNames()); + PermissionColumns columns = assignment.getColumns(); + List referenced = + columns.getColumnNames() == null + ? columns.getExcludedColumnNames() + : columns.getColumnNames(); + for (String column : referenced) { + if (!tableColumns.contains(column)) { + return new ValidationError( + ErrorResponse.RESOURCE_TYPE_COLUMN, + column, + "Permission column does not exist.", + 404); + } + } + return null; + } + + private static boolean notExpired(PermissionAssignment assignment) { + return assignment.getExpireTime() == null + || java.time.Instant.now() + .isBefore(java.time.Instant.parse(assignment.getExpireTime())); + } + + static final class ValidationError { + final String resourceType; + final String resourceName; + final String message; + final int code; + + private ValidationError( + String resourceType, String resourceName, String message, int code) { + this.resourceType = resourceType; + this.resourceName = resourceName; + this.message = message; + this.code = code; + } + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStore.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStore.java new file mode 100644 index 000000000000..83d28bc6010b --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStore.java @@ -0,0 +1,114 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.management.PermissionAssignment; +import org.apache.paimon.management.PermissionResource; + +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** Atomic permission assignment store for the REST catalog test server. */ +final class RESTPermissionStore { + + private final Map assignments = new ConcurrentHashMap<>(); + + void put(PermissionAssignment assignment) { + assignments.put(PermissionKey.fromAssignment(assignment), assignment); + } + + void remove(PermissionResource resource, String access, String principal) { + assignments.remove(new PermissionKey(resource, access, principal)); + } + + List list(PermissionResource target, Map parameters) { + return assignments.values().stream() + .filter(assignment -> assignment.getResource().equals(target)) + .filter(assignment -> matches(parameters, "principal", assignment.getPrincipal())) + .filter(assignment -> matches(parameters, "access", assignment.getAccess())) + .sorted(Comparator.comparing(RESTPermissionStore::sortKey)) + .collect(Collectors.toList()); + } + + private static boolean matches(Map parameters, String key, String value) { + return !parameters.containsKey(key) || parameters.get(key).equals(value); + } + + private static String sortKey(PermissionAssignment assignment) { + PermissionResource source = assignment.getResource(); + return source.getType().name() + + '\0' + + value(source.getDatabase()) + + '\0' + + value(source.getTable()) + + '\0' + + value(source.getFunction()) + + '\0' + + value(source.getView()) + + '\0' + + assignment.getAccess() + + '\0' + + assignment.getPrincipal(); + } + + private static String value(String value) { + return value == null ? "" : value; + } + + private static class PermissionKey { + + private final PermissionResource resource; + private final String access; + private final String principal; + + private PermissionKey(PermissionResource resource, String access, String principal) { + this.resource = resource; + this.access = access; + this.principal = principal; + } + + private static PermissionKey fromAssignment(PermissionAssignment assignment) { + return new PermissionKey( + assignment.getResource(), assignment.getAccess(), assignment.getPrincipal()); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof PermissionKey)) { + return false; + } + PermissionKey that = (PermissionKey) o; + return resource.equals(that.resource) + && access.equals(that.access) + && principal.equals(that.principal); + } + + @Override + public int hashCode() { + return Objects.hash(resource, access, principal); + } + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStoreTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStoreTest.java new file mode 100644 index 000000000000..73dfacb12976 --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTPermissionStoreTest.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import org.apache.paimon.management.PermissionAssignment; +import org.apache.paimon.management.PermissionColumns; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.ResourceType; + +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.IntStream; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests atomic replacement and exact-resource filtering in {@link RESTPermissionStore}. */ +class RESTPermissionStoreTest { + + private static final String ANALYST = "analyst"; + + @Test + void testConcurrentGrantReplacesTheSameIdentity() { + RESTPermissionStore store = new RESTPermissionStore(); + PermissionResource table = tableResource(); + + IntStream.range(0, 1000) + .parallel() + .forEach( + i -> + store.put( + new PermissionAssignment( + table, + "SELECT", + ANALYST, + Instant.ofEpochSecond(i).toString()))); + + assertThat(store.list(table, tableParameters())).hasSize(1); + } + + @Test + void testListReturnsOnlyTheExactTarget() { + RESTPermissionStore store = new RESTPermissionStore(); + PermissionResource catalog = + new PermissionResource(ResourceType.CATALOG, null, null, null, null); + store.put(new PermissionAssignment(catalog, "CREATEDATABASE", ANALYST, null)); + store.put(new PermissionAssignment(tableResource(), "SELECT", ANALYST, null)); + + assertThat(store.list(tableResource(), tableParameters())) + .singleElement() + .extracting(PermissionAssignment::getAccess) + .isEqualTo("SELECT"); + } + + @Test + void testColumnGrantReplacesTheWholeColumnRangeForTheSameIdentity() { + RESTPermissionStore store = new RESTPermissionStore(); + PermissionResource column = columnResource(); + store.put( + new PermissionAssignment( + column, + "SELECT", + ANALYST, + new PermissionColumns(Arrays.asList("id", "region"), null), + null)); + store.put( + new PermissionAssignment( + column, + "SELECT", + ANALYST, + new PermissionColumns(null, Arrays.asList("email")), + null)); + + assertThat(store.list(column, columnParameters())) + .singleElement() + .extracting(PermissionAssignment::getColumns) + .extracting(PermissionColumns::getExcludedColumnNames) + .isEqualTo(Arrays.asList("email")); + } + + private static PermissionResource tableResource() { + return new PermissionResource(ResourceType.TABLE, "sales", "orders", null, null); + } + + private static PermissionResource columnResource() { + return new PermissionResource(ResourceType.COLUMN, "sales", "orders", null, null); + } + + private static Map tableParameters() { + Map parameters = new HashMap<>(); + parameters.put("resourceType", "TABLE"); + parameters.put("database", "sales"); + parameters.put("table", "orders"); + return parameters; + } + + private static Map columnParameters() { + Map parameters = new HashMap<>(); + parameters.put("resourceType", "COLUMN"); + parameters.put("database", "sales"); + parameters.put("table", "orders"); + return parameters; + } +} diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java index e3979c437a35..bba45b64c92e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/ResourcePathsTest.java @@ -18,9 +18,13 @@ package org.apache.paimon.rest; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.ResourceType; + import org.junit.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; /** Test for {@link ResourcePaths}. */ public class ResourcePathsTest { @@ -39,4 +43,29 @@ public void testUrlEncode() { "/v1/paimon%2Faaaa/databases/test_db/tables/test_table%24snapshot", resourcePaths.table(database, objectName)); } + + @Test + public void testPermissionManagementUsesPrefix() { + ResourcePaths resourcePaths = new ResourcePaths("catalog/id"); + assertEquals("/v1/catalog%2Fid/permissions", resourcePaths.permissions()); + assertEquals("/v1/catalog%2Fid/permissions/grant", resourcePaths.grantPermission()); + assertEquals("/v1/catalog%2Fid/permissions/revoke", resourcePaths.revokePermission()); + } + + @Test + public void testPoliciesAreNestedUnderAttachmentResource() { + ResourcePaths paths = new ResourcePaths("catalog/id"); + PermissionResource catalog = + new PermissionResource(ResourceType.CATALOG, null, null, null, null); + PermissionResource database = + new PermissionResource(ResourceType.DATABASE, "sales db", null, null, null); + PermissionResource table = + new PermissionResource(ResourceType.TABLE, "sales db", "orders/all", null, null); + + assertThrows(IllegalArgumentException.class, () -> paths.policies(catalog)); + assertThrows(IllegalArgumentException.class, () -> paths.policies(database)); + assertEquals( + "/v1/catalog%2Fid/databases/sales+db/tables/orders%2Fall/policies", + paths.policies(table)); + } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/TableLifecycleLocks.java b/paimon-core/src/test/java/org/apache/paimon/rest/TableLifecycleLocks.java new file mode 100644 index 000000000000..450e9883ff3b --- /dev/null +++ b/paimon-core/src/test/java/org/apache/paimon/rest/TableLifecycleLocks.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.rest; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** Stable, deadlock-ordered locks for table-name lifecycle operations in the test server. */ +final class TableLifecycleLocks { + + private final Map locks = new ConcurrentHashMap<>(); + + Object lock(String tableName) { + return locks.computeIfAbsent(tableName, ignored -> new Object()); + } + + Object[] ordered(String left, String right) { + return left.compareTo(right) <= 0 + ? new Object[] {lock(left), lock(right)} + : new Object[] {lock(right), lock(left)}; + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/antlr4/org.apache.spark.sql.catalyst.parser.extensions/PaimonSqlExtensions.g4 b/paimon-spark/paimon-spark-common/src/main/antlr4/org.apache.spark.sql.catalyst.parser.extensions/PaimonSqlExtensions.g4 index 8c2e45b34ec2..bbd2728d9d91 100644 --- a/paimon-spark/paimon-spark-common/src/main/antlr4/org.apache.spark.sql.catalyst.parser.extensions/PaimonSqlExtensions.g4 +++ b/paimon-spark/paimon-spark-common/src/main/antlr4/org.apache.spark.sql.catalyst.parser.extensions/PaimonSqlExtensions.g4 @@ -167,6 +167,7 @@ overwriteClause expression : constant | stringMap + | stringArray ; constant @@ -180,6 +181,10 @@ stringMap : MAP '(' constant (',' constant)* ')' ; +stringArray + : ARRAY '(' (constant (',' constant)*)? ')' + ; + booleanValue : TRUE | FALSE ; @@ -214,7 +219,7 @@ nonReserved : ALTER | AS | CALL | CREATE | DAYS | DELETE | EXISTS | HOURS | IF | LIKE | NOT | OF | OR | TABLE | REPLACE | RETAIN | VERSION | TAG | TRUE | FALSE - | MAP + | ARRAY | MAP | COPY | INTO | FROM | FILE_FORMAT | PATTERN | FORCE | ON_ERROR | ABORT_STATEMENT | CONTINUE | SKIP_FILE | OVERWRITE | CSV | JSON @@ -249,6 +254,7 @@ TRUE: 'TRUE'; FALSE: 'FALSE'; MAP: 'MAP'; +ARRAY: 'ARRAY'; COPY: 'COPY'; INTO: 'INTO'; diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java index 60b5747e3db7..866a54e802a9 100644 --- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkProcedures.java @@ -29,16 +29,21 @@ import org.apache.paimon.spark.procedure.CreateBranchProcedure; import org.apache.paimon.spark.procedure.CreateFunctionProcedure; import org.apache.paimon.spark.procedure.CreateGlobalIndexProcedure; +import org.apache.paimon.spark.procedure.CreatePolicyProcedure; import org.apache.paimon.spark.procedure.CreateTagFromTimestampProcedure; import org.apache.paimon.spark.procedure.CreateTagProcedure; import org.apache.paimon.spark.procedure.DeleteBranchProcedure; import org.apache.paimon.spark.procedure.DeleteTagProcedure; import org.apache.paimon.spark.procedure.DropFunctionProcedure; import org.apache.paimon.spark.procedure.DropGlobalIndexProcedure; +import org.apache.paimon.spark.procedure.DropPolicyProcedure; import org.apache.paimon.spark.procedure.ExpirePartitionsProcedure; import org.apache.paimon.spark.procedure.ExpireSnapshotsProcedure; import org.apache.paimon.spark.procedure.ExpireTagsProcedure; import org.apache.paimon.spark.procedure.FastForwardProcedure; +import org.apache.paimon.spark.procedure.GrantPermissionProcedure; +import org.apache.paimon.spark.procedure.ListPermissionsProcedure; +import org.apache.paimon.spark.procedure.ListPoliciesProcedure; import org.apache.paimon.spark.procedure.MarkPartitionDoneProcedure; import org.apache.paimon.spark.procedure.MaterializeDeletionVectorsProcedure; import org.apache.paimon.spark.procedure.MergeBranchProcedure; @@ -57,6 +62,7 @@ import org.apache.paimon.spark.procedure.ReplaceTagProcedure; import org.apache.paimon.spark.procedure.RescaleProcedure; import org.apache.paimon.spark.procedure.ResetConsumerProcedure; +import org.apache.paimon.spark.procedure.RevokePermissionProcedure; import org.apache.paimon.spark.procedure.RewriteFileIndexProcedure; import org.apache.paimon.spark.procedure.RollbackProcedure; import org.apache.paimon.spark.procedure.RollbackToTimestampProcedure; @@ -134,6 +140,12 @@ private static Map> initProcedureBuilders() { procedureBuilders.put("rewrite_file_index", RewriteFileIndexProcedure::builder); procedureBuilders.put("copy", CopyFilesProcedure::builder); procedureBuilders.put("reassign_row_id", ReassignRowIdProcedure::builder); + procedureBuilders.put("grant_permission", GrantPermissionProcedure::builder); + procedureBuilders.put("revoke_permission", RevokePermissionProcedure::builder); + procedureBuilders.put("list_permissions", ListPermissionsProcedure::builder); + procedureBuilders.put("create_policy", CreatePolicyProcedure::builder); + procedureBuilders.put("drop_policy", DropPolicyProcedure::builder); + procedureBuilders.put("list_policies", ListPoliciesProcedure::builder); return procedureBuilders.build(); } } diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/BasePermissionProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/BasePermissionProcedure.java new file mode 100644 index 000000000000..8a4ba0b069f7 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/BasePermissionProcedure.java @@ -0,0 +1,130 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.DelegateCatalog; +import org.apache.paimon.management.PermissionAssignment; +import org.apache.paimon.management.PermissionColumns; +import org.apache.paimon.management.PermissionManagement; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.PolicyManagement; +import org.apache.paimon.management.ResourceType; +import org.apache.paimon.rest.RESTCatalog; +import org.apache.paimon.spark.catalog.WithPaimonCatalog; + +import org.apache.spark.sql.connector.catalog.TableCatalog; + +import javax.annotation.Nullable; + +import java.util.Arrays; +import java.util.Locale; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Shared REST catalog lookup and argument validation for management procedures. */ +abstract class BasePermissionProcedure extends BaseProcedure { + + protected BasePermissionProcedure(TableCatalog tableCatalog) { + super(tableCatalog); + } + + protected PermissionManagement permissionManagement() { + return restCatalog().permissionManagement(); + } + + protected PolicyManagement policyManagement() { + return restCatalog().policyManagement(); + } + + private RESTCatalog restCatalog() { + checkArgument( + tableCatalog() instanceof WithPaimonCatalog, + "Catalog '%s' is not a Paimon catalog.", + tableCatalog().name()); + Catalog root = + DelegateCatalog.rootCatalog(((WithPaimonCatalog) tableCatalog()).paimonCatalog()); + checkArgument( + root instanceof RESTCatalog, + "Catalog '%s' does not support permission or policy management.", + tableCatalog().name()); + return (RESTCatalog) root; + } + + protected static PermissionAssignment assignment( + ResourceType resourceType, + String access, + String principal, + @Nullable String database, + @Nullable String table, + @Nullable String function, + @Nullable String view, + @Nullable PermissionColumns columns, + @Nullable String expireTime) { + return new PermissionAssignment( + resource(resourceType, database, table, function, view), + access, + principal, + columns, + emptyToNull(expireTime)); + } + + protected static PermissionResource resource( + ResourceType resourceType, + @Nullable String database, + @Nullable String table, + @Nullable String function, + @Nullable String view) { + return new PermissionResource( + resourceType, + emptyToNull(database), + emptyToNull(table), + emptyToNull(function), + emptyToNull(view)); + } + + protected static > E enumValue( + String value, Class enumClass, String argument) { + checkArgument(!isBlank(value), "%s cannot be empty.", argument); + try { + return Enum.valueOf(enumClass, value.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException( + String.format( + "Invalid %s '%s'. Expected one of %s.", + argument, value, Arrays.toString(enumClass.getEnumConstants())), + e); + } + } + + @Nullable + protected static > E optionalEnum( + @Nullable String value, Class enumClass, String argument) { + return isBlank(value) ? null : enumValue(value, enumClass, argument); + } + + @Nullable + protected static String emptyToNull(@Nullable String value) { + return isBlank(value) ? null : value; + } + + protected static boolean isBlank(@Nullable String value) { + return value == null || value.trim().isEmpty(); + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/BasePolicyProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/BasePolicyProcedure.java new file mode 100644 index 000000000000..e96d31df7749 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/BasePolicyProcedure.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.management.ColumnMask; +import org.apache.paimon.management.DataPolicy; +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.PolicyType; +import org.apache.paimon.management.ResourceType; +import org.apache.paimon.management.RowFilter; + +import org.apache.spark.sql.connector.catalog.TableCatalog; + +import javax.annotation.Nullable; + +import static org.apache.paimon.utils.Preconditions.checkArgument; + +/** Shared typed argument conversion for table policy procedures. */ +abstract class BasePolicyProcedure extends BasePermissionProcedure { + + protected BasePolicyProcedure(TableCatalog tableCatalog) { + super(tableCatalog); + } + + protected static DataPolicy policy( + String database, + String table, + PolicyType policyType, + String principal, + @Nullable String predicate, + @Nullable String onColumn, + @Nullable String transform) { + PermissionResource resource = tableResource(database, table); + if (policyType == PolicyType.ROW_FILTER) { + checkArgument(isBlank(onColumn), "ROW_FILTER policy cannot specify on_column."); + checkArgument(isBlank(transform), "ROW_FILTER policy cannot specify transform."); + return DataPolicy.rowFilter(resource, new RowFilter(predicate), principal); + } + checkArgument(isBlank(predicate), "COLUMN_MASKING policy cannot specify predicate."); + return DataPolicy.columnMask(resource, new ColumnMask(onColumn, transform), principal); + } + + protected static PermissionResource tableResource(String database, String table) { + return resource(ResourceType.TABLE, database, table, null, null); + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CreatePolicyProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CreatePolicyProcedure.java new file mode 100644 index 000000000000..52b56a55faf9 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CreatePolicyProcedure.java @@ -0,0 +1,100 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.management.DataPolicy; +import org.apache.paimon.management.PolicyManagement.PolicyAlreadyExistException; +import org.apache.paimon.management.PolicyType; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; + +import static org.apache.spark.sql.types.DataTypes.StringType; + +/** Creates a table row-filter or column-mask policy. */ +public class CreatePolicyProcedure extends BasePolicyProcedure { + + private static final ProcedureParameter[] PARAMETERS = + new ProcedureParameter[] { + ProcedureParameter.required("database", StringType), + ProcedureParameter.required("table", StringType), + ProcedureParameter.required("policy_type", StringType), + ProcedureParameter.required("principal", StringType), + ProcedureParameter.optional("predicate_json", StringType), + ProcedureParameter.optional("on_column", StringType), + ProcedureParameter.optional("transform_json", StringType) + }; + + private static final StructType OUTPUT_TYPE = + new StructType( + new StructField[] { + new StructField("result", DataTypes.BooleanType, false, Metadata.empty()) + }); + + private CreatePolicyProcedure(TableCatalog tableCatalog) { + super(tableCatalog); + } + + @Override + public ProcedureParameter[] parameters() { + return PARAMETERS; + } + + @Override + public StructType outputType() { + return OUTPUT_TYPE; + } + + @Override + public InternalRow[] call(InternalRow args) { + DataPolicy policy = + policy( + args.getString(0), + args.getString(1), + enumValue(args.getString(2), PolicyType.class, PARAMETERS[2].name()), + args.getString(3), + args.isNullAt(4) ? null : args.getString(4), + args.isNullAt(5) ? null : args.getString(5), + args.isNullAt(6) ? null : args.getString(6)); + try { + policyManagement().createPolicy(policy); + } catch (PolicyAlreadyExistException e) { + throw new RuntimeException(e); + } + return new InternalRow[] {newInternalRow(true)}; + } + + public static ProcedureBuilder builder() { + return new Builder() { + @Override + protected CreatePolicyProcedure doBuild() { + return new CreatePolicyProcedure(tableCatalog()); + } + }; + } + + @Override + public String description() { + return "CreatePolicyProcedure"; + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DropPolicyProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DropPolicyProcedure.java new file mode 100644 index 000000000000..71780d1a84f9 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DropPolicyProcedure.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.PolicyType; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; + +import static org.apache.spark.sql.types.DataTypes.BooleanType; +import static org.apache.spark.sql.types.DataTypes.StringType; + +/** Drops one principal's row-filter or column-masking policy. */ +public class DropPolicyProcedure extends BasePolicyProcedure { + + private static final ProcedureParameter[] PARAMETERS = + new ProcedureParameter[] { + ProcedureParameter.required("database", StringType), + ProcedureParameter.required("table", StringType), + ProcedureParameter.required("policy_type", StringType), + ProcedureParameter.required("principal", StringType), + ProcedureParameter.optional("column", StringType), + ProcedureParameter.optional("if_exists", BooleanType) + }; + + private static final StructType OUTPUT_TYPE = + new StructType( + new StructField[] { + new StructField("result", DataTypes.BooleanType, false, Metadata.empty()) + }); + + private DropPolicyProcedure(TableCatalog tableCatalog) { + super(tableCatalog); + } + + @Override + public ProcedureParameter[] parameters() { + return PARAMETERS; + } + + @Override + public StructType outputType() { + return OUTPUT_TYPE; + } + + @Override + public InternalRow[] call(InternalRow args) { + PermissionResource resource = tableResource(args.getString(0), args.getString(1)); + PolicyType type = enumValue(args.getString(2), PolicyType.class, PARAMETERS[2].name()); + policyManagement() + .dropPolicy( + resource, + type, + args.getString(3), + args.isNullAt(4) ? null : args.getString(4), + !args.isNullAt(5) && args.getBoolean(5)); + return new InternalRow[] {newInternalRow(true)}; + } + + public static ProcedureBuilder builder() { + return new Builder() { + @Override + protected DropPolicyProcedure doBuild() { + return new DropPolicyProcedure(tableCatalog()); + } + }; + } + + @Override + public String description() { + return "DropPolicyProcedure"; + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/GrantPermissionProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/GrantPermissionProcedure.java new file mode 100644 index 000000000000..14cbef3b7c1e --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/GrantPermissionProcedure.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.management.PermissionAssignment; +import org.apache.paimon.management.PermissionColumns; +import org.apache.paimon.management.ResourceType; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.catalyst.util.ArrayData; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.unsafe.types.UTF8String; + +import javax.annotation.Nullable; + +import java.util.ArrayList; +import java.util.List; + +import static org.apache.spark.sql.types.DataTypes.StringType; +import static org.apache.spark.sql.types.DataTypes.createArrayType; + +/** Grants a permission through a REST catalog. */ +public class GrantPermissionProcedure extends BasePermissionProcedure { + + private static final ProcedureParameter[] PARAMETERS = + new ProcedureParameter[] { + ProcedureParameter.required("resource_type", StringType), + ProcedureParameter.required("access", StringType), + ProcedureParameter.required("principal", StringType), + ProcedureParameter.optional("database", StringType), + ProcedureParameter.optional("table", StringType), + ProcedureParameter.optional("function", StringType), + ProcedureParameter.optional("view", StringType), + ProcedureParameter.optional("expire_time", StringType), + ProcedureParameter.optional("column_names", createArrayType(StringType)), + ProcedureParameter.optional("excluded_column_names", createArrayType(StringType)) + }; + + private static final StructType OUTPUT_TYPE = + new StructType( + new StructField[] { + new StructField( + "result", + org.apache.spark.sql.types.DataTypes.BooleanType, + false, + Metadata.empty()) + }); + + private GrantPermissionProcedure(TableCatalog tableCatalog) { + super(tableCatalog); + } + + @Override + public ProcedureParameter[] parameters() { + return PARAMETERS; + } + + @Override + public StructType outputType() { + return OUTPUT_TYPE; + } + + @Override + public InternalRow[] call(InternalRow args) { + ResourceType resourceType = + enumValue(args.getString(0), ResourceType.class, PARAMETERS[0].name()); + PermissionAssignment assignment = + assignment( + resourceType, + args.getString(1), + args.getString(2), + args.isNullAt(3) ? null : args.getString(3), + args.isNullAt(4) ? null : args.getString(4), + args.isNullAt(5) ? null : args.getString(5), + args.isNullAt(6) ? null : args.getString(6), + columns(args, 8, 9), + args.isNullAt(7) ? null : args.getString(7)); + + permissionManagement().grantPermission(assignment); + return new InternalRow[] {newInternalRow(true)}; + } + + @Nullable + private static PermissionColumns columns( + InternalRow args, int columnNamesPos, int excludedColumnNamesPos) { + List columnNames = stringArray(args, columnNamesPos); + List excludedColumnNames = stringArray(args, excludedColumnNamesPos); + return columnNames == null && excludedColumnNames == null + ? null + : new PermissionColumns(columnNames, excludedColumnNames); + } + + @Nullable + private static List stringArray(InternalRow args, int position) { + if (args.isNullAt(position)) { + return null; + } + ArrayData array = args.getArray(position); + List values = new ArrayList<>(array.numElements()); + for (int i = 0; i < array.numElements(); i++) { + UTF8String value = array.isNullAt(i) ? null : array.getUTF8String(i); + values.add(value == null ? null : value.toString()); + } + return values; + } + + public static ProcedureBuilder builder() { + return new Builder() { + @Override + protected GrantPermissionProcedure doBuild() { + return new GrantPermissionProcedure(tableCatalog()); + } + }; + } + + @Override + public String description() { + return "GrantPermissionProcedure"; + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ListPermissionsProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ListPermissionsProcedure.java new file mode 100644 index 000000000000..70d256e5c99c --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ListPermissionsProcedure.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.PagedList; +import org.apache.paimon.management.ListPermissionsRequest; +import org.apache.paimon.management.PermissionAssignment; +import org.apache.paimon.management.PermissionColumns; +import org.apache.paimon.management.ResourceType; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.catalyst.util.GenericArrayData; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.unsafe.types.UTF8String; + +import java.util.List; + +import static org.apache.spark.sql.types.DataTypes.IntegerType; +import static org.apache.spark.sql.types.DataTypes.StringType; +import static org.apache.spark.sql.types.DataTypes.createArrayType; + +/** Lists direct permissions on an exact target. */ +public class ListPermissionsProcedure extends BasePermissionProcedure { + + private static final ProcedureParameter[] PARAMETERS = + new ProcedureParameter[] { + ProcedureParameter.required("resource_type", StringType), + ProcedureParameter.optional("database", StringType), + ProcedureParameter.optional("table", StringType), + ProcedureParameter.optional("function", StringType), + ProcedureParameter.optional("view", StringType), + ProcedureParameter.optional("principal", StringType), + ProcedureParameter.optional("access", StringType), + ProcedureParameter.optional("max_results", IntegerType), + ProcedureParameter.optional("page_token", StringType) + }; + + private static final StructType OUTPUT_TYPE = + new StructType( + new StructField[] { + field("resource_type", StringType, false), + field("database", StringType, true), + field("table", StringType, true), + field("function", StringType, true), + field("view", StringType, true), + field("access", StringType, false), + field("principal", StringType, false), + field("column_names", createArrayType(StringType), true), + field("excluded_column_names", createArrayType(StringType), true), + field("expire_time", StringType, true), + field("next_page_token", StringType, true) + }); + + private ListPermissionsProcedure(TableCatalog tableCatalog) { + super(tableCatalog); + } + + @Override + public ProcedureParameter[] parameters() { + return PARAMETERS; + } + + @Override + public StructType outputType() { + return OUTPUT_TYPE; + } + + @Override + public InternalRow[] call(InternalRow args) { + ResourceType resourceType = + enumValue(args.getString(0), ResourceType.class, PARAMETERS[0].name()); + Integer maxResults = args.isNullAt(7) ? null : args.getInt(7); + ListPermissionsRequest request = + new ListPermissionsRequest( + resourceType, + args.isNullAt(1) ? null : emptyToNull(args.getString(1)), + args.isNullAt(2) ? null : emptyToNull(args.getString(2)), + args.isNullAt(3) ? null : emptyToNull(args.getString(3)), + args.isNullAt(4) ? null : emptyToNull(args.getString(4)), + args.isNullAt(5) ? null : emptyToNull(args.getString(5)), + args.isNullAt(6) ? null : emptyToNull(args.getString(6)), + args.isNullAt(8) ? null : emptyToNull(args.getString(8)), + maxResults); + PagedList page = permissionManagement().listPermissions(request); + List assignments = page.getElements(); + if (assignments == null || assignments.isEmpty()) { + return new InternalRow[0]; + } + + InternalRow[] rows = new InternalRow[assignments.size()]; + for (int i = 0; i < assignments.size(); i++) { + PermissionAssignment assignment = assignments.get(i); + PermissionColumns columns = assignment.getColumns(); + rows[i] = + newInternalRow( + string(assignment.getResource().getType().name()), + string(assignment.getResource().getDatabase()), + string(assignment.getResource().getTable()), + string(assignment.getResource().getFunction()), + string(assignment.getResource().getView()), + string(assignment.getAccess()), + string(assignment.getPrincipal()), + stringArray(columns == null ? null : columns.getColumnNames()), + stringArray(columns == null ? null : columns.getExcludedColumnNames()), + string(assignment.getExpireTime()), + string(page.getNextPageToken())); + } + return rows; + } + + private static StructField field( + String name, org.apache.spark.sql.types.DataType type, boolean nullable) { + return new StructField(name, type, nullable, Metadata.empty()); + } + + private static UTF8String string(String value) { + return value == null ? null : UTF8String.fromString(value); + } + + private static GenericArrayData stringArray(List values) { + if (values == null) { + return null; + } + return new GenericArrayData( + values.stream().map(ListPermissionsProcedure::string).toArray()); + } + + public static ProcedureBuilder builder() { + return new Builder() { + @Override + protected ListPermissionsProcedure doBuild() { + return new ListPermissionsProcedure(tableCatalog()); + } + }; + } + + @Override + public String description() { + return "ListPermissionsProcedure"; + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ListPoliciesProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ListPoliciesProcedure.java new file mode 100644 index 000000000000..fa4d8a244fff --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/ListPoliciesProcedure.java @@ -0,0 +1,140 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.PagedList; +import org.apache.paimon.management.ColumnMask; +import org.apache.paimon.management.DataPolicy; +import org.apache.paimon.management.ListPoliciesRequest; +import org.apache.paimon.management.PolicyType; +import org.apache.paimon.management.RowFilter; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; +import org.apache.spark.unsafe.types.UTF8String; + +import java.util.List; + +import static org.apache.spark.sql.types.DataTypes.IntegerType; +import static org.apache.spark.sql.types.DataTypes.StringType; + +/** Lists policies attached to an exact table. */ +public class ListPoliciesProcedure extends BasePolicyProcedure { + + private static final ProcedureParameter[] PARAMETERS = + new ProcedureParameter[] { + ProcedureParameter.required("database", StringType), + ProcedureParameter.required("table", StringType), + ProcedureParameter.optional("policy_type", StringType), + ProcedureParameter.optional("principal", StringType), + ProcedureParameter.optional("column", StringType), + ProcedureParameter.optional("max_results", IntegerType), + ProcedureParameter.optional("page_token", StringType) + }; + + private static final StructType OUTPUT_TYPE = + new StructType( + new StructField[] { + field("database", StringType, false), + field("table", StringType, false), + field("policy_type", StringType, false), + field("principal", StringType, false), + field("predicate_json", StringType, true), + field("on_column", StringType, true), + field("transform_json", StringType, true), + field("next_page_token", StringType, true) + }); + + private ListPoliciesProcedure(TableCatalog tableCatalog) { + super(tableCatalog); + } + + @Override + public ProcedureParameter[] parameters() { + return PARAMETERS; + } + + @Override + public StructType outputType() { + return OUTPUT_TYPE; + } + + @Override + public InternalRow[] call(InternalRow args) { + ListPoliciesRequest request = + new ListPoliciesRequest( + tableResource(args.getString(0), args.getString(1)), + optionalEnum( + args.isNullAt(2) ? null : args.getString(2), + PolicyType.class, + PARAMETERS[2].name()), + args.isNullAt(3) ? null : args.getString(3), + args.isNullAt(4) ? null : args.getString(4), + args.isNullAt(6) ? null : args.getString(6), + args.isNullAt(5) ? null : args.getInt(5)); + PagedList page = policyManagement().listPolicies(request); + List policies = page.getElements(); + if (policies == null || policies.isEmpty()) { + return new InternalRow[0]; + } + InternalRow[] rows = new InternalRow[policies.size()]; + for (int i = 0; i < policies.size(); i++) { + DataPolicy policy = policies.get(i); + RowFilter rowFilter = policy.getRowFilter(); + ColumnMask columnMask = policy.getColumnMask(); + rows[i] = + newInternalRow( + string(policy.getResource().getDatabase()), + string(policy.getResource().getTable()), + string(policy.type().name()), + string(policy.getPrincipal()), + string(rowFilter == null ? null : rowFilter.getPredicate()), + string(columnMask == null ? null : columnMask.getOnColumn()), + string(columnMask == null ? null : columnMask.getTransform()), + string(page.getNextPageToken())); + } + return rows; + } + + private static StructField field( + String name, org.apache.spark.sql.types.DataType type, boolean nullable) { + return new StructField(name, type, nullable, Metadata.empty()); + } + + private static UTF8String string(String value) { + return value == null ? null : UTF8String.fromString(value); + } + + public static ProcedureBuilder builder() { + return new Builder() { + @Override + protected ListPoliciesProcedure doBuild() { + return new ListPoliciesProcedure(tableCatalog()); + } + }; + } + + @Override + public String description() { + return "ListPoliciesProcedure"; + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RevokePermissionProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RevokePermissionProcedure.java new file mode 100644 index 000000000000..516c8d1ae126 --- /dev/null +++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/RevokePermissionProcedure.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure; + +import org.apache.paimon.management.PermissionResource; +import org.apache.paimon.management.ResourceType; + +import org.apache.spark.sql.catalyst.InternalRow; +import org.apache.spark.sql.connector.catalog.TableCatalog; +import org.apache.spark.sql.types.DataTypes; +import org.apache.spark.sql.types.Metadata; +import org.apache.spark.sql.types.StructField; +import org.apache.spark.sql.types.StructType; + +import static org.apache.spark.sql.types.DataTypes.StringType; + +/** Revokes a permission by its resource identity. */ +public class RevokePermissionProcedure extends BasePermissionProcedure { + + private static final ProcedureParameter[] PARAMETERS = + new ProcedureParameter[] { + ProcedureParameter.required("resource_type", StringType), + ProcedureParameter.required("access", StringType), + ProcedureParameter.required("principal", StringType), + ProcedureParameter.optional("database", StringType), + ProcedureParameter.optional("table", StringType), + ProcedureParameter.optional("function", StringType), + ProcedureParameter.optional("view", StringType) + }; + + private static final StructType OUTPUT_TYPE = + new StructType( + new StructField[] { + new StructField("result", DataTypes.BooleanType, false, Metadata.empty()) + }); + + private RevokePermissionProcedure(TableCatalog tableCatalog) { + super(tableCatalog); + } + + @Override + public ProcedureParameter[] parameters() { + return PARAMETERS; + } + + @Override + public StructType outputType() { + return OUTPUT_TYPE; + } + + @Override + public InternalRow[] call(InternalRow args) { + ResourceType resourceType = + enumValue(args.getString(0), ResourceType.class, PARAMETERS[0].name()); + PermissionResource resource = + resource( + resourceType, + args.isNullAt(3) ? null : args.getString(3), + args.isNullAt(4) ? null : args.getString(4), + args.isNullAt(5) ? null : args.getString(5), + args.isNullAt(6) ? null : args.getString(6)); + + permissionManagement().revokePermission(resource, args.getString(1), args.getString(2)); + return new InternalRow[] {newInternalRow(true)}; + } + + public static ProcedureBuilder builder() { + return new Builder() { + @Override + protected RevokePermissionProcedure doBuild() { + return new RevokePermissionProcedure(tableCatalog()); + } + }; + } + + @Override + public String description() { + return "RevokePermissionProcedure"; + } +} diff --git a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/AbstractPaimonSparkSqlExtensionsParser.scala b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/AbstractPaimonSparkSqlExtensionsParser.scala index 7770143c8012..6cede203d4b0 100644 --- a/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/AbstractPaimonSparkSqlExtensionsParser.scala +++ b/paimon-spark/paimon-spark-common/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/AbstractPaimonSparkSqlExtensionsParser.scala @@ -71,6 +71,7 @@ abstract class AbstractPaimonSparkSqlExtensionsParser(val delegate: ParserInterf PaimonSqlExtensionsParser.TAG, PaimonSqlExtensionsParser.TRUE, PaimonSqlExtensionsParser.FALSE, + PaimonSqlExtensionsParser.ARRAY, PaimonSqlExtensionsParser.MAP, PaimonSqlExtensionsParser.COPY, PaimonSqlExtensionsParser.INTO, diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonSparkTestWithRestCatalogBase.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonSparkTestWithRestCatalogBase.scala index b8b8d53e21b0..12b5c8febecc 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonSparkTestWithRestCatalogBase.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/PaimonSparkTestWithRestCatalogBase.scala @@ -30,7 +30,7 @@ import java.util.UUID class PaimonSparkTestWithRestCatalogBase extends PaimonSparkTestBase { - private var restCatalogServer: RESTCatalogServer = _ + protected var restCatalogServer: RESTCatalogServer = _ private var serverUrl: String = _ protected var warehouse: String = _ private val initToken = "init_token" @@ -43,13 +43,25 @@ class PaimonSparkTestWithRestCatalogBase extends PaimonSparkTestBase { "paimon", CatalogOptions.WAREHOUSE.key, warehouse), - ImmutableMap.of()) + ImmutableMap.of() + ) val authProvider = new BearTokenAuthProvider(initToken) restCatalogServer = new RESTCatalogServer(tempDBDir.getCanonicalPath, authProvider, config, warehouse) restCatalogServer.start() serverUrl = restCatalogServer.getUrl super.beforeAll() + Seq("analyst", "first", "second", "reader", "function_reader").foreach( + restCatalogServer.registerManagementPrincipal) + restCatalogServer.registerManagementPrincipal("analysts") + restCatalogServer.registerManagementPrincipal("admin") + spark.sql("CREATE DATABASE IF NOT EXISTS paimon.sales") + spark.sql("""CREATE TABLE IF NOT EXISTS paimon.sales.orders ( + | id INT, + | region STRING, + | email STRING) + |TBLPROPERTIES ('query-auth.enabled' = 'true') + |""".stripMargin) } override protected def afterAll(): Unit = { diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/PermissionProcedureTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/PermissionProcedureTest.scala new file mode 100644 index 000000000000..261324dbe52c --- /dev/null +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/procedure/PermissionProcedureTest.scala @@ -0,0 +1,681 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.spark.procedure + +import org.apache.paimon.catalog.Identifier +import org.apache.paimon.data.BinaryString +import org.apache.paimon.management.{PermissionResource, ResourceType} +import org.apache.paimon.predicate.{ConcatTransform, Equal, FieldRef, FieldTransform, LeafPredicate} +import org.apache.paimon.spark.{PaimonSparkTestBase, PaimonSparkTestWithRestCatalogBase} +import org.apache.paimon.types.DataTypes +import org.apache.paimon.utils.JsonSerdeUtil + +import org.apache.spark.sql.Row +import org.assertj.core.api.Assertions.assertThat + +import java.util.{Arrays, Collections} + +/** End-to-end tests for permission and policy management procedures. */ +class PermissionProcedureTest extends PaimonSparkTestWithRestCatalogBase { + + test("grant, list and idempotently revoke a permission") { + checkAnswer( + spark.sql("""CALL sys.grant_permission( + | resource_type => 'table', + | access => 'select', + | principal => 'analyst', + | database => 'sales', + | table => 'orders', + | expire_time => '2027-01-01T00:00:00Z') + |""".stripMargin), + Row(true) + ) + + checkAnswer( + spark.sql("""CALL sys.grant_permission( + | resource_type => 'TABLE', + | access => 'SELECT', + | principal => 'analyst', + | database => 'sales', + | table => 'orders', + | expire_time => '2028-01-01T00:00:00Z') + |""".stripMargin), + Row(true) + ) + + val listed = spark.sql("""CALL sys.list_permissions( + | resource_type => 'TABLE', + | database => 'sales', + | table => 'orders', + | principal => 'analyst') + |""".stripMargin) + assertThat(listed.columns).containsExactly( + "resource_type", + "database", + "table", + "function", + "view", + "access", + "principal", + "column_names", + "excluded_column_names", + "expire_time", + "next_page_token") + val assignments = listed.collect() + assertThat(assignments).hasSize(1) + val assignment = assignments.head + assertThat(assignment.getString(0)).isEqualTo("TABLE") + assertThat(assignment.getString(1)).isEqualTo("sales") + assertThat(assignment.getString(2)).isEqualTo("orders") + assertThat(assignment.getString(5)).isEqualTo("SELECT") + assertThat(assignment.getString(6)).isEqualTo("analyst") + assertThat(assignment.isNullAt(7)).isTrue + assertThat(assignment.isNullAt(8)).isTrue + assertThat(assignment.getString(9)).isEqualTo("2028-01-01T00:00:00Z") + assertThat(assignment.isNullAt(10)).isTrue + + val revoke = """CALL sys.revoke_permission( + | resource_type => 'TABLE', + | access => 'SELECT', + | principal => 'analyst', + | database => 'sales', + | table => 'orders') + |""".stripMargin + checkAnswer(spark.sql(revoke), Row(true)) + checkAnswer(spark.sql(revoke), Row(true)) + checkAnswer( + spark.sql("""CALL sys.list_permissions( + | resource_type => 'TABLE', + | database => 'sales', + | table => 'orders', + | principal => 'analyst') + |""".stripMargin), + Nil + ) + } + + test("list permissions supports opaque pagination tokens") { + grantCatalogPermission("first") + grantCatalogPermission("second") + + val first = spark + .sql("CALL sys.list_permissions(resource_type => 'CATALOG', max_results => 1)") + .head() + assertThat(first.getString(6)).isEqualTo("first") + assertThat(first.getString(10)).isEqualTo("1") + + val second = spark + .sql( + "CALL sys.list_permissions(resource_type => 'CATALOG', max_results => 1, page_token => '1')" + ) + .head() + assertThat(second.getString(6)).isEqualTo("second") + assertThat(second.isNullAt(10)).isTrue + } + + test("grant and list explicit descendant scopes") { + checkAnswer( + spark.sql("""CALL sys.grant_permission( + | resource_type => 'CATALOG_ALL', + | access => 'SELECT', + | principal => 'analyst') + |""".stripMargin), + Row(true) + ) + val catalogAll = spark + .sql( + "CALL sys.list_permissions(resource_type => 'CATALOG_ALL', principal => 'analyst')" + ) + .head() + assertThat(catalogAll.getString(0)).isEqualTo("CATALOG_ALL") + assertThat(catalogAll.isNullAt(1)).isTrue + assertThat(catalogAll.getString(5)).isEqualTo("SELECT") + + checkAnswer( + spark.sql("""CALL sys.grant_permission( + | resource_type => 'DATABASE_ALL', + | database => 'sales', + | access => 'UPDATE', + | principal => 'analyst') + |""".stripMargin), + Row(true) + ) + val databaseAll = spark + .sql("""CALL sys.list_permissions( + | resource_type => 'DATABASE_ALL', + | database => 'sales', + | principal => 'analyst') + |""".stripMargin) + .head() + assertThat(databaseAll.getString(0)).isEqualTo("DATABASE_ALL") + assertThat(databaseAll.getString(1)).isEqualTo("sales") + assertThat(databaseAll.isNullAt(2)).isTrue + assertThat(databaseAll.getString(5)).isEqualTo("UPDATE") + } + + test("grant, replace, list and enforce column permissions") { + restCatalogServer.setQueryPrincipals(Collections.singleton("analyst")) + + checkAnswer( + spark.sql("""CALL sys.grant_permission( + | resource_type => 'COLUMN', + | access => 'SELECT', + | principal => 'analyst', + | database => 'sales', + | table => 'orders', + | column_names => array('id', 'region')) + |""".stripMargin), + Row(true) + ) + + val included = spark.sql("""CALL sys.list_permissions( + | resource_type => 'COLUMN', + | database => 'sales', + | table => 'orders', + | principal => 'analyst') + |""".stripMargin) + assertThat(included.columns).containsExactly( + "resource_type", + "database", + "table", + "function", + "view", + "access", + "principal", + "column_names", + "excluded_column_names", + "expire_time", + "next_page_token") + assertThat(included.head().getSeq[String](7)).isEqualTo(Seq("id", "region")) + assertThat(included.head().isNullAt(8)).isTrue + + checkAnswer(spark.sql("SELECT id, region FROM paimon.sales.orders"), Nil) + val deniedEmail = intercept[Exception] { + spark.sql("SELECT email FROM paimon.sales.orders").collect() + } + assertThat(deniedEmail.getMessage).contains("permission") + + checkAnswer( + spark.sql("""CALL sys.grant_permission( + | resource_type => 'COLUMN', + | access => 'SELECT', + | principal => 'analyst', + | database => 'sales', + | table => 'orders', + | excluded_column_names => array('region')) + |""".stripMargin), + Row(true) + ) + checkAnswer(spark.sql("SELECT id, email FROM paimon.sales.orders"), Nil) + val deniedRegion = intercept[Exception] { + spark.sql("SELECT region FROM paimon.sales.orders").collect() + } + assertThat(deniedRegion.getMessage).contains("permission") + + restCatalogServer.registerManagementPrincipal("limited") + restCatalogServer.setQueryPrincipals(new java.util.HashSet(Arrays.asList("analyst", "limited"))) + checkAnswer( + spark.sql("""CALL sys.grant_permission( + | resource_type => 'COLUMN', access => 'SELECT', + | principal => 'limited', database => 'sales', table => 'orders', + | column_names => array('id', 'region')) + |""".stripMargin), + Row(true) + ) + checkAnswer(spark.sql("SELECT id FROM paimon.sales.orders"), Nil) + val deniedByIntersection = intercept[Exception] { + spark.sql("SELECT email FROM paimon.sales.orders").collect() + } + assertThat(deniedByIntersection.getMessage).contains("permission") + + checkAnswer( + spark.sql("""CALL sys.revoke_permission( + | resource_type => 'COLUMN', access => 'SELECT', + | principal => 'analyst', database => 'sales', table => 'orders') + |""".stripMargin), + Row(true) + ) + } + + test("column permission validates query authorization and referenced columns") { + spark.sql("CREATE TABLE paimon.sales.disabled_columns (id INT)") + val disabled = intercept[Exception] { + spark + .sql("""CALL sys.grant_permission( + | resource_type => 'COLUMN', access => 'SELECT', + | principal => 'analyst', database => 'sales', table => 'disabled_columns', + | column_names => array('id')) + |""".stripMargin) + .collect() + } + assertThat(disabled.getMessage).contains("query-auth.enabled=true") + + val missing = intercept[Exception] { + spark + .sql("""CALL sys.grant_permission( + | resource_type => 'COLUMN', access => 'SELECT', + | principal => 'analyst', database => 'sales', table => 'orders', + | excluded_column_names => array('missing')) + |""".stripMargin) + .collect() + } + assertThat(missing.getMessage).contains("Permission column does not exist") + } + + test("create, reject duplicates, apply and idempotently drop table data policies") { + restCatalogServer.setQueryPrincipals(Collections.singleton("analyst")) + spark.sql( + "INSERT OVERWRITE paimon.sales.orders VALUES " + + "(1, 'APAC', 'apac@example.com'), (2, 'EMEA', 'emea@example.com')") + + val apacFilter = stringEqualsPredicate(1, "region", "APAC") + val emeaFilter = stringEqualsPredicate(1, "region", "EMEA") + val emailMask = concatFieldTransform(1, "region", "-masked") + + checkAnswer( + spark.sql(s"""CALL sys.create_policy( + | database => 'sales', + | table => 'orders', + | policy_type => 'ROW_FILTER', + | principal => 'analyst', + | predicate_json => ${sqlLiteral(apacFilter)}) + |""".stripMargin), + Row(true) + ) + + val listed = spark.sql("""CALL sys.list_policies( + | database => 'sales', + | table => 'orders', + | policy_type => 'ROW_FILTER', + | principal => 'analyst') + |""".stripMargin) + assertThat(listed.columns).containsExactly( + "database", + "table", + "policy_type", + "principal", + "predicate_json", + "on_column", + "transform_json", + "next_page_token") + val direct = listed.head() + assertThat(direct.getString(2)).isEqualTo("ROW_FILTER") + assertThat(direct.getString(3)).isEqualTo("analyst") + assertThat(direct.getString(4)).contains("\"name\":\"region\"") + assertThat(direct.isNullAt(5)).isTrue + assertThat(direct.isNullAt(6)).isTrue + checkAnswer( + spark.sql("SELECT id, region, email FROM paimon.sales.orders ORDER BY id"), + Row(1, "APAC", "apac@example.com") + ) + + val duplicate = intercept[Exception] { + spark + .sql(s"""CALL sys.create_policy( + | database => 'sales', + | table => 'orders', + | policy_type => 'ROW_FILTER', + | principal => 'analyst', + | predicate_json => ${sqlLiteral(emeaFilter)}) + |""".stripMargin) + .collect() + } + assertThat(duplicate.getMessage).contains("already exists") + val unchanged = spark + .sql("""CALL sys.list_policies( + | database => 'sales', + | table => 'orders', + | policy_type => 'ROW_FILTER', + | principal => 'analyst') + |""".stripMargin) + .head() + assertThat(unchanged.getString(4)).contains("APAC").doesNotContain("EMEA") + + checkAnswer( + spark.sql(s"""CALL sys.create_policy( + | database => 'sales', + | table => 'orders', + | policy_type => 'COLUMN_MASKING', + | principal => 'analyst', + | on_column => 'email', + | transform_json => ${sqlLiteral(emailMask)}) + |""".stripMargin), + Row(true) + ) + + val mask = spark + .sql("""CALL sys.list_policies( + | database => 'sales', + | table => 'orders', + | policy_type => 'COLUMN_MASKING') + |""".stripMargin) + .head() + assertThat(mask.getString(2)).isEqualTo("COLUMN_MASKING") + assertThat(mask.getString(3)).isEqualTo("analyst") + assertThat(mask.isNullAt(4)).isTrue + assertThat(mask.getString(5)).isEqualTo("email") + assertThat(mask.getString(6)).contains("-masked") + + checkAnswer( + spark.sql("SELECT id, region, email FROM paimon.sales.orders ORDER BY id"), + Row(1, "APAC", "APAC-masked") + ) + checkAnswer( + spark.sql("SELECT email FROM paimon.sales.orders"), + Row("APAC-masked") + ) + checkAnswer( + spark.sql("""CALL sys.drop_policy( + | database => 'sales', + | table => 'orders', + | policy_type => 'ROW_FILTER', + | principal => 'analyst', + | if_exists => true) + |""".stripMargin), + Row(true) + ) + checkAnswer( + spark.sql("""CALL sys.drop_policy( + | database => 'sales', + | table => 'orders', + | policy_type => 'ROW_FILTER', + | principal => 'analyst', + | if_exists => true) + |""".stripMargin), + Row(true) + ) + checkAnswer( + spark.sql("""CALL sys.list_policies( + | database => 'sales', + | table => 'orders', + | policy_type => 'ROW_FILTER', + | principal => 'analyst') + |""".stripMargin), + Nil + ) + spark.sql( + "CALL sys.drop_policy(database => 'sales', table => 'orders', " + + "policy_type => 'COLUMN_MASKING', " + + "principal => 'analyst', column => 'email', if_exists => true)") + } + + test("policy creation validates query authorization, JSON and columns") { + val validFilter = intEqualsPredicate(0, "id", 1) + spark.sql("CREATE TABLE paimon.sales.disabled_orders (id INT)") + + val disabled = intercept[Exception] { + spark + .sql(s"""CALL sys.create_policy( + | database => 'sales', table => 'disabled_orders', + | policy_type => 'ROW_FILTER', predicate_json => ${sqlLiteral(validFilter)}, + | principal => 'analyst') + |""".stripMargin) + .collect() + } + assertThat(disabled.getMessage).contains("query-auth.enabled=true") + + val malformed = intercept[Exception] { + spark + .sql("""CALL sys.create_policy( + | database => 'sales', table => 'orders', + | policy_type => 'ROW_FILTER', predicate_json => '{bad', + | principal => 'analyst') + |""".stripMargin) + .collect() + } + assertThat(malformed.getMessage).contains("Unexpected character") + + val unknownColumnFilter = stringEqualsPredicate(0, "unknown", "APAC") + val missingColumn = intercept[Exception] { + spark + .sql(s"""CALL sys.create_policy( + | database => 'sales', table => 'orders', + | policy_type => 'ROW_FILTER', predicate_json => ${sqlLiteral(unknownColumnFilter)}, + | principal => 'analyst') + |""".stripMargin) + .collect() + } + assertThat(missingColumn.getMessage).contains("column unknown") + + val missingPrincipal = intercept[Exception] { + spark + .sql(s"""CALL sys.create_policy( + | database => 'sales', table => 'orders', + | policy_type => 'ROW_FILTER', predicate_json => ${sqlLiteral(validFilter)}, + | principal => 'missing') + |""".stripMargin) + .collect() + } + assertThat(missingPrincipal.getMessage).contains("principal does not exist") + + val jsonNull = intercept[Exception] { + spark + .sql("""CALL sys.create_policy( + | database => 'sales', table => 'orders', + | policy_type => 'ROW_FILTER', predicate_json => 'null', + | principal => 'analyst') + |""".stripMargin) + .collect() + } + assertThat(jsonNull.getMessage).contains("JSON null") + + val mixedDefinition = intercept[Exception] { + spark + .sql(s"""CALL sys.create_policy( + | database => 'sales', table => 'orders', + | policy_type => 'ROW_FILTER', predicate_json => ${sqlLiteral(validFilter)}, + | transform_json => ${sqlLiteral(constantStringTransform("****"))}, + | principal => 'analyst') + |""".stripMargin) + .collect() + } + assertThat(mixedDefinition.getMessage).contains("cannot specify transform") + } + + test("list policies supports opaque pagination tokens") { + val filter = intEqualsPredicate(0, "id", 1) + Seq("first", "second").foreach { + principal => + checkAnswer( + spark.sql(s"""CALL sys.create_policy( + | database => 'sales', table => 'orders', + | policy_type => 'ROW_FILTER', principal => '$principal', + | predicate_json => ${sqlLiteral(filter)}) + |""".stripMargin), + Row(true) + ) + } + try { + val first = spark + .sql("CALL sys.list_policies(database => 'sales', table => 'orders', max_results => 1)") + .head() + assertThat(first.getString(3)).isEqualTo("first") + assertThat(first.getString(7)).isEqualTo("1") + + val second = spark + .sql( + "CALL sys.list_policies(database => 'sales', table => 'orders', " + + "max_results => 1, page_token => '1')") + .head() + assertThat(second.getString(3)).isEqualTo("second") + assertThat(second.isNullAt(7)).isTrue + } finally { + Seq("first", "second").foreach { + principal => + spark.sql( + "CALL sys.drop_policy(database => 'sales', table => 'orders', " + + s"policy_type => 'ROW_FILTER', principal => '$principal', if_exists => true)") + } + } + } + + test("table lifecycle preserves policy enforcement and rejects unsafe schema changes") { + restCatalogServer.setQueryPrincipals(Collections.singleton("analyst")) + spark.sql("""CREATE TABLE paimon.sales.lifecycle_orders ( + | id INT, + | region STRING) + |TBLPROPERTIES ('query-auth.enabled' = 'true') + |""".stripMargin) + spark.sql("INSERT INTO paimon.sales.lifecycle_orders VALUES (1, 'APAC'), (2, 'EMEA')") + val lifecycleFilter = stringEqualsPredicate(1, "region", "APAC") + checkAnswer( + spark.sql(s"""CALL sys.create_policy( + | database => 'sales', table => 'lifecycle_orders', + | policy_type => 'ROW_FILTER', predicate_json => ${sqlLiteral(lifecycleFilter)}, + | principal => 'analyst') + |""".stripMargin), + Row(true) + ) + + spark.sql("ALTER TABLE paimon.sales.lifecycle_orders RENAME TO paimon.sales.renamed_orders") + assertThat( + spark + .sql("CALL sys.list_policies(database => 'sales', table => 'renamed_orders', " + + "policy_type => 'ROW_FILTER', principal => 'analyst')") + .head() + .getString(1)).isEqualTo("renamed_orders") + assertThat( + paimonCatalog + .authTableQuery(Identifier.create("sales", "renamed_orders"), null) + .extractPredicate()).isNotNull + + val disableAuth = intercept[Exception] { + spark + .sql("ALTER TABLE paimon.sales.renamed_orders " + + "SET TBLPROPERTIES ('query-auth.enabled' = 'false')") + .collect() + } + assertThat(disableAuth.getMessage).contains("Cannot disable query-auth.enabled") + + val renameColumn = intercept[Exception] { + spark + .sql("ALTER TABLE paimon.sales.renamed_orders RENAME COLUMN region TO area") + .collect() + } + assertThat(renameColumn.getMessage).contains("column region") + + spark.sql("DROP TABLE paimon.sales.renamed_orders") + spark.sql("""CREATE TABLE paimon.sales.renamed_orders ( + | id INT, + | region STRING) + |TBLPROPERTIES ('query-auth.enabled' = 'true') + |""".stripMargin) + spark.sql("INSERT INTO paimon.sales.renamed_orders VALUES (1, 'APAC'), (2, 'EMEA')") + checkAnswer( + spark.sql("SELECT id, region FROM paimon.sales.renamed_orders ORDER BY id"), + Seq(Row(1, "APAC"), Row(2, "EMEA")) + ) + checkAnswer( + spark.sql( + "CALL sys.list_policies(database => 'sales', table => 'renamed_orders', " + + "policy_type => 'ROW_FILTER', principal => 'analyst')"), + Nil + ) + assertThat( + paimonCatalog + .authTableQuery(Identifier.create("sales", "renamed_orders"), null) + .extractPredicate()).isNull + } + + test("management endpoints enforce target authorization") { + val resource = + new PermissionResource(ResourceType.TABLE, "sales", "orders", null, null) + restCatalogServer.denyManagementPermission(resource) + try { + val permissionError = intercept[Exception] { + spark + .sql("""CALL sys.grant_permission( + | resource_type => 'TABLE', access => 'SELECT', + | principal => 'analyst', + | database => 'sales', table => 'orders') + |""".stripMargin) + .collect() + } + assertThat(permissionError.getMessage).contains("cannot manage permissions") + + val policyError = intercept[Exception] { + spark + .sql("CALL sys.list_policies(database => 'sales', table => 'orders')") + .collect() + } + assertThat(policyError.getMessage).contains("cannot manage permissions") + } finally { + restCatalogServer.allowManagementPermission(resource) + } + } + + private def grantCatalogPermission(principal: String): Unit = { + checkAnswer( + spark.sql(s"""CALL sys.grant_permission( + | resource_type => 'CATALOG', + | access => 'CREATEDATABASE', + | principal => '$principal') + |""".stripMargin), + Row(true) + ) + } + + private def stringEqualsPredicate(index: Int, column: String, constant: String): String = { + JsonSerdeUtil.toFlatJson( + LeafPredicate.of( + new FieldTransform(new FieldRef(index, column, DataTypes.STRING())), + Equal.INSTANCE, + Collections.singletonList(BinaryString.fromString(constant)))) + } + + private def intEqualsPredicate(index: Int, column: String, constant: Int): String = { + JsonSerdeUtil.toFlatJson( + LeafPredicate.of( + new FieldTransform(new FieldRef(index, column, DataTypes.INT())), + Equal.INSTANCE, + Collections.singletonList(Integer.valueOf(constant)))) + } + + private def constantStringTransform(constant: String): String = { + JsonSerdeUtil.toFlatJson( + new ConcatTransform(Collections.singletonList(BinaryString.fromString(constant)))) + } + + private def concatFieldTransform(index: Int, column: String, suffix: String): String = { + JsonSerdeUtil.toFlatJson( + new ConcatTransform(Arrays + .asList(new FieldRef(index, column, DataTypes.STRING()), BinaryString.fromString(suffix)))) + } + + private def sqlLiteral(value: String): String = { + "'" + value.replace("'", "''") + "'" + } +} + +/** Management procedures must fail clearly for catalogs without the REST capability. */ +class PermissionProcedureUnsupportedCatalogTest extends PaimonSparkTestBase { + + test("filesystem catalog does not expose permission management") { + val error = intercept[IllegalArgumentException] { + spark + .sql("""CALL sys.grant_permission( + | resource_type => 'CATALOG', + | access => 'CREATEDATABASE', + | principal => 'admin') + |""".stripMargin) + .collect() + } + assertThat(error.getMessage).contains("does not support permission or policy management") + } +} diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogQualifiedCreateTableLikeTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogQualifiedCreateTableLikeTest.scala index c4eb2cd6443a..350462fc1ca0 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogQualifiedCreateTableLikeTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/sql/CatalogQualifiedCreateTableLikeTest.scala @@ -139,6 +139,11 @@ class CatalogQualifiedCreateTableLikeTest extends PaimonSparkTestBase { Assertions.assertEquals("skip_file", skipFileCommand.targetIdent.name()) Assertions.assertEquals(Seq("test"), skipFileCommand.targetIdent.namespace().toSeq) + val arrayCommand = + parseCreateTableLikeCommand("CREATE TABLE paimon.test.array LIKE paimon.test.source_tbl") + Assertions.assertEquals("array", arrayCommand.targetIdent.name()) + Assertions.assertEquals(Seq("test"), arrayCommand.targetIdent.namespace().toSeq) + val nestedIdentifierCommand = parseCreateTableLikeCommand( "CREATE TABLE paimon.test.extra.target_tbl LIKE paimon.test.extra.source_tbl")