diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml index 0e53ae7b11..05b1f025cb 100644 --- a/.github/workflows/test-pr.yaml +++ b/.github/workflows/test-pr.yaml @@ -144,16 +144,16 @@ jobs: if ! command -v stack > /dev/null 2>&1; then curl -sL "https://get.haskellstack.org/" | sh fi - - name: Install Python 3.7 + - name: Install Python 3.12 if: ${{ contains(matrix.fixture, 'python') }} uses: actions/setup-python@v4 with: - python-version: 3.7 + python-version: "3.12" - name: Install Python Dependencies if: ${{ contains(matrix.fixture, 'python') }} run: | - pip3.7 install mypy python-dateutil types-python-dateutil + pip3.12 install mypy python-dateutil types-python-dateutil - name: Install flow if: ${{ contains(matrix.fixture, 'flow') }} run: | diff --git a/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts b/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts index 59271b599c..20d5fc5aa8 100644 --- a/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts +++ b/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts @@ -370,7 +370,15 @@ export class JSONPythonRenderer extends PythonRenderer { ", ", this.typingDecl("x", "Any"), ")", - this.typeHint(" -> ", this.withTyping("List"), "[", tvar, "]"), + this.typeHint( + " -> ", + this.pyOptions.features.builtinGenerics + ? "list" + : this.withTyping("List"), + "[", + tvar, + "]", + ), ":", ], () => { @@ -422,7 +430,9 @@ export class JSONPythonRenderer extends PythonRenderer { ")", this.typeHint( " -> ", - this.withTyping("Dict"), + this.pyOptions.features.builtinGenerics + ? "dict" + : this.withTyping("Dict"), "[str, ", tvar, "]", @@ -611,7 +621,8 @@ export class JSONPythonRenderer extends PythonRenderer { (_integerType) => "int", (_doubleType) => "float", (_stringType) => "str", - (_arrayType) => "List", + (_arrayType) => + this.pyOptions.features.builtinGenerics ? "list" : "List", (classType) => this.nameForNamedType(classType), (_mapType) => "dict", (enumType) => this.nameForNamedType(enumType), diff --git a/packages/quicktype-core/src/language/Python/PythonRenderer.ts b/packages/quicktype-core/src/language/Python/PythonRenderer.ts index 09072ca05f..e0d60263ab 100644 --- a/packages/quicktype-core/src/language/Python/PythonRenderer.ts +++ b/packages/quicktype-core/src/language/Python/PythonRenderer.ts @@ -1,6 +1,7 @@ import { arrayIntercalate, iterableFirst, + iterableSome, mapSortBy, mapUpdateInto, setUnionInto, @@ -19,17 +20,15 @@ import { defined, panic } from "../../support/Support.js"; import type { TargetLanguage } from "../../TargetLanguage.js"; import { followTargetType } from "../../Transformers.js"; import { + ArrayType, type ClassProperty, ClassType, EnumType, + MapType, type Type, UnionType, } from "../../Type/index.js"; -import { - matchType, - nullableFromUnion, - removeNullFromUnion, -} from "../../Type/TypeUtils.js"; +import { matchType, removeNullFromUnion } from "../../Type/TypeUtils.js"; import { forbiddenPropertyNames, forbiddenTypeNames } from "./constants.js"; import type { pythonOptions } from "./language.js"; @@ -137,56 +136,160 @@ export class PythonRenderer extends ConvenienceRenderer { return this.withImport("typing", name); } - protected namedType(t: Type): Sourcelike { + protected namedType(t: Type, suppressQuotes = false): Sourcelike { const name = this.nameForNamedType(t); - if (this.declaredTypes.has(t)) return name; + if (suppressQuotes || this.declaredTypes.has(t)) return name; return ["'", name, "'"]; } - protected pythonType(t: Type, _isRootTypeDef = false): Sourcelike { + // Would rendering `t` as a type annotation right now require a forward + // reference, i.e. does it mention a named type that hasn't been declared + // yet? + private typeContainsForwardRef(t: Type): boolean { + const actualType = followTargetType(t); + if (actualType instanceof ClassType || actualType instanceof EnumType) { + return !this.declaredTypes.has(actualType); + } + + if (actualType instanceof ArrayType) { + return this.typeContainsForwardRef(actualType.items); + } + + if (actualType instanceof MapType) { + return this.typeContainsForwardRef(actualType.values); + } + + if (actualType instanceof UnionType) { + return iterableSome(actualType.members, (m) => + this.typeContainsForwardRef(m), + ); + } + + return false; + } + + // Renders a union with PEP 604 syntax: `A | B | None`. A quoted forward + // reference is not allowed as an operand of `|` (`'A' | None` is a runtime + // `TypeError`), so if any member needs a forward reference we quote the + // whole union expression instead, suppressing the quotes on the individual + // names. A `" = None"` default, if required, must go outside the closing + // quote: `foo: 'Foo | None' = None`. + private pep604UnionType( + unionType: UnionType, + isRootTypeDef: boolean, + suppressQuotes: boolean, + ): Sourcelike { + const [hasNull, nonNulls] = removeNullFromUnion(unionType); + const needsQuotes = + !suppressQuotes && + iterableSome(nonNulls, (m) => this.typeContainsForwardRef(m)); + const quote = needsQuotes ? "'" : ""; + const memberTypes = Array.from(nonNulls).map((m) => + this.pythonType(m, false, true), + ); + + const union: Sourcelike[] = [ + quote, + arrayIntercalate(" | ", memberTypes), + ]; + if (hasNull !== null) { + union.push(" | None"); + } + + union.push(quote); + + if (hasNull !== null) { + union.push(...this.noneDefault(isRootTypeDef)); + } + + return union; + } + + // A `" = None"` default for a class property whose value can be `None`. + // Only emitted for root level type defs, otherwise we may get type defs + // like `List[Optional[int] = None]`, which are invalid. Every property + // that gets a default must sort after all properties that don't — see + // `sortClassProperties`. + private noneDefault(isRootTypeDef: boolean): string[] { + if ( + isRootTypeDef && + !this.getAlphabetizeProperties() && + (this.pyOptions.features.dataClasses || + this.pyOptions.pydanticBaseModel) + ) { + return [" = None"]; + } + + return []; + } + + // Does `pythonType(p.type, true)` end in a `" = None"` default? This + // must mirror the `noneDefault` calls in `pythonType` exactly: nullable + // unions, plus `Any` and `None` typed properties — an optional `Any` + // stays `Any` (`null` is absorbed by it), and an optional `null` + // collapses to just `null`, so those also default to `None`. + private classPropertyHasNoneDefault(p: ClassProperty): boolean { + const actualType = followTargetType(p.type); + if (actualType instanceof UnionType) { + const [hasNull] = removeNullFromUnion(actualType); + return hasNull !== null; + } + + return actualType.kind === "any" || actualType.kind === "null"; + } + + protected pythonType( + t: Type, + _isRootTypeDef = false, + suppressQuotes = false, + ): Sourcelike { const actualType = followTargetType(t); return matchType( actualType, - (_anyType) => this.withTyping("Any"), - (_nullType) => "None", + (_anyType) => [ + this.withTyping("Any"), + ...this.noneDefault(_isRootTypeDef), + ], + (_nullType) => ["None", ...this.noneDefault(_isRootTypeDef)], (_boolType) => "bool", (_integerType) => "int", (_doubletype) => "float", (_stringType) => "str", (arrayType) => [ - this.withTyping("List"), + this.pyOptions.features.builtinGenerics + ? "list" + : this.withTyping("List"), "[", - this.pythonType(arrayType.items), + this.pythonType(arrayType.items, false, suppressQuotes), "]", ], - (classType) => this.namedType(classType), + (classType) => this.namedType(classType, suppressQuotes), (mapType) => [ - this.withTyping("Dict"), + this.pyOptions.features.builtinGenerics + ? "dict" + : this.withTyping("Dict"), "[str, ", - this.pythonType(mapType.values), + this.pythonType(mapType.values, false, suppressQuotes), "]", ], - (enumType) => this.namedType(enumType), + (enumType) => this.namedType(enumType, suppressQuotes), (unionType) => { + if (this.pyOptions.features.unionOperators) { + return this.pep604UnionType( + unionType, + _isRootTypeDef, + suppressQuotes, + ); + } + const [hasNull, nonNulls] = removeNullFromUnion(unionType); const memberTypes = Array.from(nonNulls).map((m) => - this.pythonType(m), + this.pythonType(m, false, suppressQuotes), ); if (hasNull !== null) { - const rest: string[] = []; - if ( - !this.getAlphabetizeProperties() && - (this.pyOptions.features.dataClasses || - this.pyOptions.pydanticBaseModel) && - _isRootTypeDef - ) { - // Only push "= None" if this is a root level type def - // otherwise we may get type defs like List[Optional[int] = None] - // which are invalid - rest.push(" = None"); - } + const rest = this.noneDefault(_isRootTypeDef); if (nonNulls.size > 1) { this.withImport("typing", "Union"); @@ -321,13 +424,11 @@ export class PythonRenderer extends ConvenienceRenderer { this.pyOptions.features.dataClasses || this.pyOptions.pydanticBaseModel ) { - return mapSortBy(properties, (p: ClassProperty) => { - return (p.type instanceof UnionType && - nullableFromUnion(p.type) !== null) || - p.isOptional - ? 1 - : 0; - }); + // Properties that get a `" = None"` default must come after all + // properties that don't, or the generated dataclass is invalid. + return mapSortBy(properties, (p: ClassProperty) => + this.classPropertyHasNoneDefault(p) ? 1 : 0, + ); } return super.sortClassProperties(properties, propertyNames); diff --git a/packages/quicktype-core/src/language/Python/constants.ts b/packages/quicktype-core/src/language/Python/constants.ts index d32c109235..1c96606977 100644 --- a/packages/quicktype-core/src/language/Python/constants.ts +++ b/packages/quicktype-core/src/language/Python/constants.ts @@ -5,7 +5,9 @@ export const forbiddenTypeNames = [ "None", "Enum", "List", + "list", "Dict", + "dict", "Optional", "Union", "Iterable", diff --git a/packages/quicktype-core/src/language/Python/language.ts b/packages/quicktype-core/src/language/Python/language.ts index 101c5115b8..263026b7e5 100644 --- a/packages/quicktype-core/src/language/Python/language.ts +++ b/packages/quicktype-core/src/language/Python/language.ts @@ -21,8 +21,12 @@ import { JSONPythonRenderer } from "./JSONPythonRenderer.js"; import { PythonRenderer } from "./PythonRenderer.js"; export interface PythonFeatures { + /** PEP 585 builtin generics (`list[str]`, `dict[str, int]`), Python 3.9+ */ + builtinGenerics: boolean; dataClasses: boolean; typeHints: boolean; + /** PEP 604 union operators (`str | None`), Python 3.10+ */ + unionOperators: boolean; } export const pythonOptions = { @@ -30,11 +34,38 @@ export const pythonOptions = { "python-version", "Python version", { - "3.5": { typeHints: false, dataClasses: false }, - "3.6": { typeHints: true, dataClasses: false }, - "3.7": { typeHints: true, dataClasses: true }, - }, - "3.6", + "3.5": { + typeHints: false, + dataClasses: false, + builtinGenerics: false, + unionOperators: false, + }, + "3.6": { + typeHints: true, + dataClasses: false, + builtinGenerics: false, + unionOperators: false, + }, + "3.7": { + typeHints: true, + dataClasses: true, + builtinGenerics: false, + unionOperators: false, + }, + "3.9": { + typeHints: true, + dataClasses: true, + builtinGenerics: true, + unionOperators: false, + }, + "3.10": { + typeHints: true, + dataClasses: true, + builtinGenerics: true, + unionOperators: true, + }, + } satisfies Record, + "3.10", ), justTypes: new BooleanOption("just-types", "Classes only", false), nicePropertyNames: new BooleanOption( diff --git a/test/fixtures/python/run.sh b/test/fixtures/python/run.sh index 0ad6e67872..011cad464c 100755 --- a/test/fixtures/python/run.sh +++ b/test/fixtures/python/run.sh @@ -3,7 +3,7 @@ if [ "x$QUICKTYPE_PYTHON_VERSION" = "x2.7" ] ; then PYTHON="python2.7" else - PYTHON="python3.7" + PYTHON="python3.12" fi "$PYTHON" "$@" diff --git a/test/languages.ts b/test/languages.ts index 94f9828e3a..b42cd0b4ab 100644 --- a/test/languages.ts +++ b/test/languages.ts @@ -263,7 +263,13 @@ export const PythonLanguage: Language = { "keyword-unions.schema", // Requires more than 255 arguments ], rendererOptions: {}, - quickTestRendererOptions: [{ "python-version": "3.5" }], + quickTestRendererOptions: [ + // The default is "3.10"; keep the older feature sets covered. + { "python-version": "3.5" }, + { "python-version": "3.6" }, + { "python-version": "3.7" }, + { "python-version": "3.9" }, + ], sourceFiles: ["src/language/Python/index.ts"], };