From cbecc7f34c7d5131da1b7182431500e4fe7a168b Mon Sep 17 00:00:00 2001 From: Matthew Lee Date: Wed, 25 Jun 2025 22:26:34 -0500 Subject: [PATCH 1/7] Support for python 3.9 --- .../src/language/Python/JSONPythonRenderer.ts | 4 +-- .../src/language/Python/PythonRenderer.ts | 25 +++++++++++++++++-- .../src/language/Python/constants.ts | 2 ++ .../src/language/Python/language.ts | 7 +++--- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts b/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts index b06965694..1e96cca0f 100644 --- a/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts +++ b/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts @@ -377,7 +377,7 @@ export class JSONPythonRenderer extends PythonRenderer { ", ", this.typingDecl("x", "Any"), ")", - this.typeHint(" -> ", this.withTyping("List"), "[", tvar, "]"), + this.typeHint(" -> ", this.pyOptions.features.builtinGenerics ? "list" : this.withTyping("List"), "[", tvar, "]"), ":", ], () => { @@ -618,7 +618,7 @@ export class JSONPythonRenderer extends PythonRenderer { (_integerType) => "int", (_doubleType) => "float", (_stringType) => "str", - (_arrayType) => "List", + (_arrayType) => this.pyOptions.features.builtinGenerics ? "list" : "List", (classType) => this.nameForNamedType(classType), (_mapType) => "dict", (enumType) => this.nameForNamedType(enumType), diff --git a/packages/quicktype-core/src/language/Python/PythonRenderer.ts b/packages/quicktype-core/src/language/Python/PythonRenderer.ts index fd7bb84e8..80592d2f2 100644 --- a/packages/quicktype-core/src/language/Python/PythonRenderer.ts +++ b/packages/quicktype-core/src/language/Python/PythonRenderer.ts @@ -162,14 +162,14 @@ export class PythonRenderer extends ConvenienceRenderer { (_doubletype) => "float", (_stringType) => "str", (arrayType) => [ - this.withTyping("List"), + this.pyOptions.features.builtinGenerics ? "list" : this.withTyping("List"), "[", this.pythonType(arrayType.items), "]", ], (classType) => this.namedType(classType), (mapType) => [ - this.withTyping("Dict"), + this.pyOptions.features.builtinGenerics ? "dict" : this.withTyping("Dict"), "[str, ", this.pythonType(mapType.values), "]", @@ -196,6 +196,14 @@ export class PythonRenderer extends ConvenienceRenderer { } if (nonNulls.size > 1) { + if (this.pyOptions.features.builtinGenerics) { + return [ + arrayIntercalate(" | ", memberTypes), + " | None", + ...rest, + ]; + } + this.withImport("typing", "Union"); return [ this.withTyping("Optional"), @@ -206,6 +214,13 @@ export class PythonRenderer extends ConvenienceRenderer { ]; } + if (this.pyOptions.features.builtinGenerics) { + return [ + defined(iterableFirst(memberTypes)), + " | None", + ...rest, + ]; + } return [ this.withTyping("Optional"), "[", @@ -215,6 +230,12 @@ export class PythonRenderer extends ConvenienceRenderer { ]; } + if (this.pyOptions.features.builtinGenerics) { + return [ + arrayIntercalate(" | ", memberTypes), + ]; + } + return [ this.withTyping("Union"), "[", diff --git a/packages/quicktype-core/src/language/Python/constants.ts b/packages/quicktype-core/src/language/Python/constants.ts index d32c10923..1c9660697 100644 --- a/packages/quicktype-core/src/language/Python/constants.ts +++ b/packages/quicktype-core/src/language/Python/constants.ts @@ -5,7 +5,9 @@ export const forbiddenTypeNames = [ "None", "Enum", "List", + "list", "Dict", + "dict", "Optional", "Union", "Iterable", diff --git a/packages/quicktype-core/src/language/Python/language.ts b/packages/quicktype-core/src/language/Python/language.ts index c6a35b0db..524435e0a 100644 --- a/packages/quicktype-core/src/language/Python/language.ts +++ b/packages/quicktype-core/src/language/Python/language.ts @@ -29,9 +29,10 @@ export const pythonOptions = { "python-version", "Python version", { - "3.5": { typeHints: false, dataClasses: false }, - "3.6": { typeHints: true, dataClasses: false }, - "3.7": { typeHints: true, dataClasses: true }, + "3.5": { typeHints: false, dataClasses: false, builtinGenerics: false }, + "3.6": { typeHints: true, dataClasses: false, builtinGenerics: false }, + "3.7": { typeHints: true, dataClasses: true, builtinGenerics: false }, + "3.9": { typeHints: true, dataClasses: true, builtinGenerics: true }, }, "3.6", ), From 2ad08bf4c4106e4b61cb3e7d377fd96b1894ba58 Mon Sep 17 00:00:00 2001 From: Matthew Lee Date: Thu, 26 Jun 2025 00:12:58 -0500 Subject: [PATCH 2/7] Fix bug --- .../quicktype-core/src/language/Python/JSONPythonRenderer.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts b/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts index 1e96cca0f..04be1538a 100644 --- a/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts +++ b/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts @@ -429,7 +429,7 @@ export class JSONPythonRenderer extends PythonRenderer { ")", this.typeHint( " -> ", - this.withTyping("Dict"), + this.pyOptions.features.builtinGenerics ? "dict" : "Dict", "[str, ", tvar, "]", @@ -620,7 +620,7 @@ export class JSONPythonRenderer extends PythonRenderer { (_stringType) => "str", (_arrayType) => this.pyOptions.features.builtinGenerics ? "list" : "List", (classType) => this.nameForNamedType(classType), - (_mapType) => "dict", + (_mapType) => this.pyOptions.features.builtinGenerics ? "dict" : "Dict", (enumType) => this.nameForNamedType(enumType), (_unionType) => undefined, (transformedStringType) => { From 6b00ecab06859bccd7b5e448e0a94c997f02cf68 Mon Sep 17 00:00:00 2001 From: Matthew Lee Date: Thu, 26 Jun 2025 00:17:07 -0500 Subject: [PATCH 3/7] Update Python version in tests to 3.9 --- .github/workflows/test-pr.yaml | 388 ++++++++++++++++----------------- test/fixtures/python/run.sh | 2 +- 2 files changed, 195 insertions(+), 195 deletions(-) diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml index d497d9b1c..5fc78c111 100644 --- a/.github/workflows/test-pr.yaml +++ b/.github/workflows/test-pr.yaml @@ -1,197 +1,197 @@ name: Test PR on: - pull_request: - branches: - - master - - "release/**" + pull_request: + branches: + - master + - "release/**" jobs: - build: - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@v3 - - uses: ./.github/workflows/setup - - test: - needs: [build] - runs-on: ${{ matrix.runs-on }} - - strategy: - fail-fast: true - - matrix: - fixture: - - typescript,typescript-zod,typescript-effect-schema - - javascript,schema-javascript - - golang,schema-golang - - cjson,schema-cjson - - cplusplus,schema-cplusplus - - flow,schema-flow - - java,schema-java - - python,schema-python - - haskell,schema-haskell - - csharp,schema-csharp,schema-json-csharp,graphql-csharp,csharp-SystemTextJson - - json-ts-csharp - - dart,schema-dart - # - swift,schema-swift # pgp issue - - javascript-prop-types - - ruby - - php - - scala3,schema-scala3 - - elixir,schema-elixir,graphql-elixir - - # Partially working - # - schema-typescript # TODO Unify with typescript once fixed - - # Implementation is too outdated to test in GitHub Actions - # - elm,schema-elm - - # Language is too niche / obscure to test easily on ubuntu-22.04 - # - pike,schema-pike - - # Not yet started - # @schani can you help me understand this fixture? - # It looks like it tests 13+ languages? - # - graphql - - # Never tested? - # - crystal - - runs-on: [ubuntu-22.04] - - include: - # Rust is very slow, so we use a larger runner - - fixture: rust,schema-rust - runs-on: ubuntu-latest-16-cores - # Kotlin is also slow - - fixture: kotlin,schema-kotlin,kotlin-jackson,schema-kotlin-jackson - runs-on: ubuntu-latest-16-cores - - # - fixture: objective-c # segfault on compiled test cmd - # runs-on: macos-latest - - name: ${{ matrix.fixture }} - steps: - - uses: actions/checkout@v3 - - uses: ./.github/workflows/setup - - - name: Setup PHP - if: ${{ contains(matrix.fixture, 'php') }} - uses: shivammathur/setup-php@v2 - with: - php-version: "8.2" - - - name: Setup Dart - if: ${{ contains(matrix.fixture, 'dart') }} - uses: dart-lang/setup-dart@v1.3 - with: - sdk: 2.14.4 - - - name: Setup Swift - if: ${{ contains(matrix.fixture, 'swift') }} - uses: swift-actions/setup-swift@v1 - with: - swift-version: "5.7.2" - - - name: Install Ruby - uses: ruby/setup-ruby@v1 - if: ${{ contains(matrix.fixture, 'ruby') }} - with: - ruby-version: "3.2.0" - - - name: Setup .NET Core SDK - if: ${{ contains(matrix.fixture, 'csharp') }} - uses: actions/setup-dotnet@v3 - with: - dotnet-version: 6 - - - name: Install Rust - if: ${{ contains(matrix.fixture, 'rust') }} - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - - name: Install Elm - if: ${{ contains(matrix.fixture, 'elm') }} - run: | - curl -L -o elm.gz https://github.com/elm/compiler/releases/download/0.19.1/binary-for-linux-64-bit.gz - gunzip elm.gz - chmod +x elm - sudo mv elm /usr/local/bin/ - - name: Install Haskell - if: ${{ contains(matrix.fixture, 'haskell') }} - run: | - if ! command -v stack > /dev/null 2>&1; then - curl -sL "https://get.haskellstack.org/" | sh - fi - - name: Install Python 3.7 - if: ${{ contains(matrix.fixture, 'python') }} - uses: actions/setup-python@v4 - with: - python-version: 3.7 - - - name: Install Python Dependencies - if: ${{ contains(matrix.fixture, 'python') }} - run: | - pip3.7 install mypy python-dateutil types-python-dateutil - - name: Install flow - if: ${{ contains(matrix.fixture, 'flow') }} - run: | - npm install -g flow-bin@0.66.0 flow-remove-types@1.2.3 - - name: Install Java - if: ${{ matrix.fixture == 'java,schema-java' || contains(matrix.fixture, 'kotlin') }} - uses: actions/setup-java@v3 - with: - java-version: "11" - distribution: "adopt" - - - name: Install Maven - if: ${{ matrix.fixture == 'java,schema-java' }} - uses: stCarolas/setup-maven@v4.5 - with: - maven-version: 3.8.2 - - - name: Install Kotlin - if: ${{ contains(matrix.fixture, 'kotlin') }} - uses: fwilhe2/setup-kotlin@main - - - name: Install go - if: ${{ contains(matrix.fixture, 'golang') }} - uses: actions/setup-go@v3 - with: - go-version: 1.15 - - - name: Install C - if: ${{ contains(matrix.fixture, 'cjson') }} - run: | - sudo apt-get update - sudo apt-get -y install build-essential valgrind - - - name: Install C++ - if: ${{ contains(matrix.fixture, 'cplusplus') }} - run: | - sudo apt-get update - sudo apt-get -y install libboost-all-dev software-properties-common g++ --assume-yes - - - name: Install scala - if: ${{ contains(matrix.fixture, 'scala3') }} - uses: VirtusLab/scala-cli-setup@main - - run: echo '@main def hello() = println("We need this spam print statement for bloop to exit correctly...")' | scala-cli _ - if: ${{ contains(matrix.fixture, 'scala3') }} - - - name: Install Elixir - if: ${{ contains(matrix.fixture, 'elixir') }} - uses: erlef/setup-beam@v1 - with: - elixir-version: "1.15.7" - otp-version: "26.0" - - - run: QUICKTEST=true FIXTURE=${{ matrix.fixture }} npm test - - test-complete: - if: ${{ cancelled() || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') }} - needs: test - runs-on: ubuntu-22.04 - steps: - - run: | - echo "Some workflows have failed!" - exit 1 + build: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v3 + - uses: ./.github/workflows/setup + + test: + needs: [build] + runs-on: ${{ matrix.runs-on }} + + strategy: + fail-fast: true + + matrix: + fixture: + - typescript,typescript-zod,typescript-effect-schema + - javascript,schema-javascript + - golang,schema-golang + - cjson,schema-cjson + - cplusplus,schema-cplusplus + - flow,schema-flow + - java,schema-java + - python,schema-python + - haskell,schema-haskell + - csharp,schema-csharp,schema-json-csharp,graphql-csharp,csharp-SystemTextJson + - json-ts-csharp + - dart,schema-dart + # - swift,schema-swift # pgp issue + - javascript-prop-types + - ruby + - php + - scala3,schema-scala3 + - elixir,schema-elixir,graphql-elixir + + # Partially working + # - schema-typescript # TODO Unify with typescript once fixed + + # Implementation is too outdated to test in GitHub Actions + # - elm,schema-elm + + # Language is too niche / obscure to test easily on ubuntu-22.04 + # - pike,schema-pike + + # Not yet started + # @schani can you help me understand this fixture? + # It looks like it tests 13+ languages? + # - graphql + + # Never tested? + # - crystal + + runs-on: [ubuntu-22.04] + + include: + # Rust is very slow, so we use a larger runner + - fixture: rust,schema-rust + runs-on: ubuntu-latest-16-cores + # Kotlin is also slow + - fixture: kotlin,schema-kotlin,kotlin-jackson,schema-kotlin-jackson + runs-on: ubuntu-latest-16-cores + + # - fixture: objective-c # segfault on compiled test cmd + # runs-on: macos-latest + + name: ${{ matrix.fixture }} + steps: + - uses: actions/checkout@v3 + - uses: ./.github/workflows/setup + + - name: Setup PHP + if: ${{ contains(matrix.fixture, 'php') }} + uses: shivammathur/setup-php@v2 + with: + php-version: "8.2" + + - name: Setup Dart + if: ${{ contains(matrix.fixture, 'dart') }} + uses: dart-lang/setup-dart@v1.3 + with: + sdk: 2.14.4 + + - name: Setup Swift + if: ${{ contains(matrix.fixture, 'swift') }} + uses: swift-actions/setup-swift@v1 + with: + swift-version: "5.7.2" + + - name: Install Ruby + uses: ruby/setup-ruby@v1 + if: ${{ contains(matrix.fixture, 'ruby') }} + with: + ruby-version: "3.2.0" + + - name: Setup .NET Core SDK + if: ${{ contains(matrix.fixture, 'csharp') }} + uses: actions/setup-dotnet@v3 + with: + dotnet-version: 6 + + - name: Install Rust + if: ${{ contains(matrix.fixture, 'rust') }} + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + + - name: Install Elm + if: ${{ contains(matrix.fixture, 'elm') }} + run: | + curl -L -o elm.gz https://github.com/elm/compiler/releases/download/0.19.1/binary-for-linux-64-bit.gz + gunzip elm.gz + chmod +x elm + sudo mv elm /usr/local/bin/ + - name: Install Haskell + if: ${{ contains(matrix.fixture, 'haskell') }} + run: | + if ! command -v stack > /dev/null 2>&1; then + curl -sL "https://get.haskellstack.org/" | sh + fi + - name: Install Python 3.9 + if: ${{ contains(matrix.fixture, 'python') }} + uses: actions/setup-python@v4 + with: + python-version: 3.9 + + - name: Install Python Dependencies + if: ${{ contains(matrix.fixture, 'python') }} + run: | + pip3.9 install mypy python-dateutil types-python-dateutil + - name: Install flow + if: ${{ contains(matrix.fixture, 'flow') }} + run: | + npm install -g flow-bin@0.66.0 flow-remove-types@1.2.3 + - name: Install Java + if: ${{ matrix.fixture == 'java,schema-java' || contains(matrix.fixture, 'kotlin') }} + uses: actions/setup-java@v3 + with: + java-version: "11" + distribution: "adopt" + + - name: Install Maven + if: ${{ matrix.fixture == 'java,schema-java' }} + uses: stCarolas/setup-maven@v4.5 + with: + maven-version: 3.8.2 + + - name: Install Kotlin + if: ${{ contains(matrix.fixture, 'kotlin') }} + uses: fwilhe2/setup-kotlin@main + + - name: Install go + if: ${{ contains(matrix.fixture, 'golang') }} + uses: actions/setup-go@v3 + with: + go-version: 1.15 + + - name: Install C + if: ${{ contains(matrix.fixture, 'cjson') }} + run: | + sudo apt-get update + sudo apt-get -y install build-essential valgrind + + - name: Install C++ + if: ${{ contains(matrix.fixture, 'cplusplus') }} + run: | + sudo apt-get update + sudo apt-get -y install libboost-all-dev software-properties-common g++ --assume-yes + + - name: Install scala + if: ${{ contains(matrix.fixture, 'scala3') }} + uses: VirtusLab/scala-cli-setup@main + - run: echo '@main def hello() = println("We need this spam print statement for bloop to exit correctly...")' | scala-cli _ + if: ${{ contains(matrix.fixture, 'scala3') }} + + - name: Install Elixir + if: ${{ contains(matrix.fixture, 'elixir') }} + uses: erlef/setup-beam@v1 + with: + elixir-version: "1.15.7" + otp-version: "26.0" + + - run: QUICKTEST=true FIXTURE=${{ matrix.fixture }} npm test + + test-complete: + if: ${{ cancelled() || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure') }} + needs: test + runs-on: ubuntu-22.04 + steps: + - run: | + echo "Some workflows have failed!" + exit 1 diff --git a/test/fixtures/python/run.sh b/test/fixtures/python/run.sh index 0ad6e6787..98c4273a0 100755 --- a/test/fixtures/python/run.sh +++ b/test/fixtures/python/run.sh @@ -3,7 +3,7 @@ if [ "x$QUICKTYPE_PYTHON_VERSION" = "x2.7" ] ; then PYTHON="python2.7" else - PYTHON="python3.7" + PYTHON="python3.9" fi "$PYTHON" "$@" From 9824c3e46aa0b96ca40b27ee8c5d4fc06fc0251d Mon Sep 17 00:00:00 2001 From: Matthew Lee Date: Thu, 26 Jun 2025 02:21:42 -0500 Subject: [PATCH 4/7] Fixed bug --- .../src/language/Python/PythonRenderer.ts | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/packages/quicktype-core/src/language/Python/PythonRenderer.ts b/packages/quicktype-core/src/language/Python/PythonRenderer.ts index 80592d2f2..f574a6dc1 100644 --- a/packages/quicktype-core/src/language/Python/PythonRenderer.ts +++ b/packages/quicktype-core/src/language/Python/PythonRenderer.ts @@ -144,13 +144,13 @@ export class PythonRenderer extends ConvenienceRenderer { return this.withImport("typing", name); } - protected namedType(t: Type): Sourcelike { + protected namedType(t: Type, _suppressQuotes: boolean = false): Sourcelike { const name = this.nameForNamedType(t); - if (this.declaredTypes.has(t)) return name; + if (_suppressQuotes || this.declaredTypes.has(t)) return name; return ["'", name, "'"]; } - protected pythonType(t: Type, _isRootTypeDef = false): Sourcelike { + protected pythonType(t: Type, _isRootTypeDef = false, _suppressQuotes = false): Sourcelike { const actualType = followTargetType(t); return matchType( @@ -167,20 +167,27 @@ export class PythonRenderer extends ConvenienceRenderer { this.pythonType(arrayType.items), "]", ], - (classType) => this.namedType(classType), + (classType) => this.namedType(classType, _suppressQuotes), (mapType) => [ this.pyOptions.features.builtinGenerics ? "dict" : this.withTyping("Dict"), "[str, ", this.pythonType(mapType.values), "]", ], - (enumType) => this.namedType(enumType), + (enumType) => this.namedType(enumType, _suppressQuotes), (unionType) => { const [hasNull, nonNulls] = removeNullFromUnion(unionType); + const hasClassOrEnumType = Array.from(nonNulls).some( + t => t instanceof ClassType || t instanceof EnumType + ); + const encapsulator = hasClassOrEnumType ? "'" : "" + const memberTypes = Array.from(nonNulls).map((m) => - this.pythonType(m), + // Suppress quotes for namedType + this.pythonType(m, false, true), ); + if (hasNull !== null) { const rest: string[] = []; if ( @@ -198,9 +205,11 @@ export class PythonRenderer extends ConvenienceRenderer { if (nonNulls.size > 1) { if (this.pyOptions.features.builtinGenerics) { return [ + encapsulator, arrayIntercalate(" | ", memberTypes), " | None", ...rest, + encapsulator ]; } @@ -216,9 +225,11 @@ export class PythonRenderer extends ConvenienceRenderer { if (this.pyOptions.features.builtinGenerics) { return [ + encapsulator, defined(iterableFirst(memberTypes)), " | None", ...rest, + encapsulator ]; } return [ @@ -232,7 +243,9 @@ export class PythonRenderer extends ConvenienceRenderer { if (this.pyOptions.features.builtinGenerics) { return [ + encapsulator, arrayIntercalate(" | ", memberTypes), + encapsulator ]; } From aba2303c959ef32b84631cb4ca8b8258ec596bc6 Mon Sep 17 00:00:00 2001 From: Matthew Lee Date: Thu, 26 Jun 2025 02:22:27 -0500 Subject: [PATCH 5/7] Use python 3.9 in tests --- packages/quicktype-core/src/language/Python/language.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/quicktype-core/src/language/Python/language.ts b/packages/quicktype-core/src/language/Python/language.ts index 524435e0a..6bac0bb73 100644 --- a/packages/quicktype-core/src/language/Python/language.ts +++ b/packages/quicktype-core/src/language/Python/language.ts @@ -34,7 +34,7 @@ export const pythonOptions = { "3.7": { typeHints: true, dataClasses: true, builtinGenerics: false }, "3.9": { typeHints: true, dataClasses: true, builtinGenerics: true }, }, - "3.6", + "3.9", ), justTypes: new BooleanOption("just-types", "Classes only", false), nicePropertyNames: new BooleanOption( From 325cb37123a2d94d09b1a42407848e710b174641 Mon Sep 17 00:00:00 2001 From: Mark Probst Date: Sun, 19 Jul 2026 18:33:04 -0400 Subject: [PATCH 6/7] Fix Python 3.10 type-hint syntax support - Rename the new python-version level to 3.10: PEP 604 unions (X | None) require Python 3.10, not 3.9. Split the feature flag into builtinGenerics (PEP 585, 3.9+) and unionOperators (PEP 604, 3.10+), which also yields a 3.9 level with builtin generics but typing Optional/Union. Make 3.10 the default python-version. - Emit " = None" defaults outside the quoted annotation: the annotation 'Foo | None = None' is invalid syntax. - Only quote PEP 604 unions when a member actually needs a forward reference, and never quote inside an already-quoted annotation. - Restore withTyping("Dict") registration in the dict converter so the typing import is emitted for pre-3.9 versions, and revert typeObject's map branch to the runtime type "dict". - Cover the legacy typing.* code paths in QUICKTEST via quickTestRendererOptions for 3.5/3.6/3.7/3.9. - Run python fixtures under Python 3.12 in CI and the fixture driver. Co-Authored-By: Claude Fable 5 --- .github/workflows/test-pr.yaml | 6 +- .../src/language/Python/JSONPythonRenderer.ts | 19 ++- .../src/language/Python/PythonRenderer.ts | 145 ++++++++++++------ .../src/language/Python/language.ts | 42 ++++- test/fixtures/python/run.sh | 2 +- test/languages.ts | 8 +- 6 files changed, 163 insertions(+), 59 deletions(-) diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml index f7eec829a..984b65a42 100644 --- a/.github/workflows/test-pr.yaml +++ b/.github/workflows/test-pr.yaml @@ -127,16 +127,16 @@ jobs: if ! command -v stack > /dev/null 2>&1; then curl -sL "https://get.haskellstack.org/" | sh fi - - name: Install Python 3.9 + - name: Install Python 3.12 if: ${{ contains(matrix.fixture, 'python') }} uses: actions/setup-python@v4 with: - python-version: 3.9 + python-version: "3.12" - name: Install Python Dependencies if: ${{ contains(matrix.fixture, 'python') }} run: | - pip3.9 install mypy python-dateutil types-python-dateutil + pip3.12 install mypy python-dateutil types-python-dateutil - name: Install flow if: ${{ contains(matrix.fixture, 'flow') }} run: | diff --git a/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts b/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts index 04be1538a..9a1842742 100644 --- a/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts +++ b/packages/quicktype-core/src/language/Python/JSONPythonRenderer.ts @@ -377,7 +377,15 @@ export class JSONPythonRenderer extends PythonRenderer { ", ", this.typingDecl("x", "Any"), ")", - this.typeHint(" -> ", this.pyOptions.features.builtinGenerics ? "list" : this.withTyping("List"), "[", tvar, "]"), + this.typeHint( + " -> ", + this.pyOptions.features.builtinGenerics + ? "list" + : this.withTyping("List"), + "[", + tvar, + "]", + ), ":", ], () => { @@ -429,7 +437,9 @@ export class JSONPythonRenderer extends PythonRenderer { ")", this.typeHint( " -> ", - this.pyOptions.features.builtinGenerics ? "dict" : "Dict", + this.pyOptions.features.builtinGenerics + ? "dict" + : this.withTyping("Dict"), "[str, ", tvar, "]", @@ -618,9 +628,10 @@ export class JSONPythonRenderer extends PythonRenderer { (_integerType) => "int", (_doubleType) => "float", (_stringType) => "str", - (_arrayType) => this.pyOptions.features.builtinGenerics ? "list" : "List", + (_arrayType) => + this.pyOptions.features.builtinGenerics ? "list" : "List", (classType) => this.nameForNamedType(classType), - (_mapType) => this.pyOptions.features.builtinGenerics ? "dict" : "Dict", + (_mapType) => "dict", (enumType) => this.nameForNamedType(enumType), (_unionType) => undefined, (transformedStringType) => { diff --git a/packages/quicktype-core/src/language/Python/PythonRenderer.ts b/packages/quicktype-core/src/language/Python/PythonRenderer.ts index 0910f8f2d..506ceea62 100644 --- a/packages/quicktype-core/src/language/Python/PythonRenderer.ts +++ b/packages/quicktype-core/src/language/Python/PythonRenderer.ts @@ -1,6 +1,7 @@ import { arrayIntercalate, iterableFirst, + iterableSome, mapSortBy, mapUpdateInto, setUnionInto, @@ -19,9 +20,11 @@ import { defined, panic } from "../../support/Support"; import type { TargetLanguage } from "../../TargetLanguage"; import { followTargetType } from "../../Transformers"; import { + ArrayType, type ClassProperty, ClassType, EnumType, + MapType, type Type, UnionType, } from "../../Type"; @@ -137,13 +140,89 @@ export class PythonRenderer extends ConvenienceRenderer { return this.withImport("typing", name); } - protected namedType(t: Type, _suppressQuotes: boolean = false): Sourcelike { + protected namedType(t: Type, suppressQuotes = false): Sourcelike { const name = this.nameForNamedType(t); - if (_suppressQuotes || this.declaredTypes.has(t)) return name; + if (suppressQuotes || this.declaredTypes.has(t)) return name; return ["'", name, "'"]; } - protected pythonType(t: Type, _isRootTypeDef = false, _suppressQuotes = false): Sourcelike { + // Would rendering `t` as a type annotation right now require a forward + // reference, i.e. does it mention a named type that hasn't been declared + // yet? + private typeContainsForwardRef(t: Type): boolean { + const actualType = followTargetType(t); + if (actualType instanceof ClassType || actualType instanceof EnumType) { + return !this.declaredTypes.has(actualType); + } + + if (actualType instanceof ArrayType) { + return this.typeContainsForwardRef(actualType.items); + } + + if (actualType instanceof MapType) { + return this.typeContainsForwardRef(actualType.values); + } + + if (actualType instanceof UnionType) { + return iterableSome(actualType.members, (m) => + this.typeContainsForwardRef(m), + ); + } + + return false; + } + + // Renders a union with PEP 604 syntax: `A | B | None`. A quoted forward + // reference is not allowed as an operand of `|` (`'A' | None` is a runtime + // `TypeError`), so if any member needs a forward reference we quote the + // whole union expression instead, suppressing the quotes on the individual + // names. A `" = None"` default, if required, must go outside the closing + // quote: `foo: 'Foo | None' = None`. + private pep604UnionType( + unionType: UnionType, + isRootTypeDef: boolean, + suppressQuotes: boolean, + ): Sourcelike { + const [hasNull, nonNulls] = removeNullFromUnion(unionType); + const needsQuotes = + !suppressQuotes && + iterableSome(nonNulls, (m) => this.typeContainsForwardRef(m)); + const quote = needsQuotes ? "'" : ""; + const memberTypes = Array.from(nonNulls).map((m) => + this.pythonType(m, false, true), + ); + + const union: Sourcelike[] = [ + quote, + arrayIntercalate(" | ", memberTypes), + ]; + if (hasNull !== null) { + union.push(" | None"); + } + + union.push(quote); + + if ( + hasNull !== null && + !this.getAlphabetizeProperties() && + (this.pyOptions.features.dataClasses || + this.pyOptions.pydanticBaseModel) && + isRootTypeDef + ) { + // Only push "= None" if this is a root level type def + // otherwise we may get type defs like list[int | None = None] + // which are invalid + union.push(" = None"); + } + + return union; + } + + protected pythonType( + t: Type, + _isRootTypeDef = false, + suppressQuotes = false, + ): Sourcelike { const actualType = followTargetType(t); return matchType( @@ -155,32 +234,37 @@ export class PythonRenderer extends ConvenienceRenderer { (_doubletype) => "float", (_stringType) => "str", (arrayType) => [ - this.pyOptions.features.builtinGenerics ? "list" : this.withTyping("List"), + this.pyOptions.features.builtinGenerics + ? "list" + : this.withTyping("List"), "[", - this.pythonType(arrayType.items), + this.pythonType(arrayType.items, false, suppressQuotes), "]", ], - (classType) => this.namedType(classType, _suppressQuotes), + (classType) => this.namedType(classType, suppressQuotes), (mapType) => [ - this.pyOptions.features.builtinGenerics ? "dict" : this.withTyping("Dict"), + this.pyOptions.features.builtinGenerics + ? "dict" + : this.withTyping("Dict"), "[str, ", - this.pythonType(mapType.values), + this.pythonType(mapType.values, false, suppressQuotes), "]", ], - (enumType) => this.namedType(enumType, _suppressQuotes), + (enumType) => this.namedType(enumType, suppressQuotes), (unionType) => { + if (this.pyOptions.features.unionOperators) { + return this.pep604UnionType( + unionType, + _isRootTypeDef, + suppressQuotes, + ); + } + const [hasNull, nonNulls] = removeNullFromUnion(unionType); - const hasClassOrEnumType = Array.from(nonNulls).some( - t => t instanceof ClassType || t instanceof EnumType - ); - const encapsulator = hasClassOrEnumType ? "'" : "" - const memberTypes = Array.from(nonNulls).map((m) => - // Suppress quotes for namedType - this.pythonType(m, false, true), + this.pythonType(m, false, suppressQuotes), ); - if (hasNull !== null) { const rest: string[] = []; if ( @@ -196,16 +280,6 @@ export class PythonRenderer extends ConvenienceRenderer { } if (nonNulls.size > 1) { - if (this.pyOptions.features.builtinGenerics) { - return [ - encapsulator, - arrayIntercalate(" | ", memberTypes), - " | None", - ...rest, - encapsulator - ]; - } - this.withImport("typing", "Union"); return [ this.withTyping("Optional"), @@ -216,15 +290,6 @@ export class PythonRenderer extends ConvenienceRenderer { ]; } - if (this.pyOptions.features.builtinGenerics) { - return [ - encapsulator, - defined(iterableFirst(memberTypes)), - " | None", - ...rest, - encapsulator - ]; - } return [ this.withTyping("Optional"), "[", @@ -234,14 +299,6 @@ export class PythonRenderer extends ConvenienceRenderer { ]; } - if (this.pyOptions.features.builtinGenerics) { - return [ - encapsulator, - arrayIntercalate(" | ", memberTypes), - encapsulator - ]; - } - return [ this.withTyping("Union"), "[", diff --git a/packages/quicktype-core/src/language/Python/language.ts b/packages/quicktype-core/src/language/Python/language.ts index 6bac0bb73..42f389aa2 100644 --- a/packages/quicktype-core/src/language/Python/language.ts +++ b/packages/quicktype-core/src/language/Python/language.ts @@ -20,8 +20,12 @@ import { JSONPythonRenderer } from "./JSONPythonRenderer"; import { PythonRenderer } from "./PythonRenderer"; export interface PythonFeatures { + /** PEP 585 builtin generics (`list[str]`, `dict[str, int]`), Python 3.9+ */ + builtinGenerics: boolean; dataClasses: boolean; typeHints: boolean; + /** PEP 604 union operators (`str | None`), Python 3.10+ */ + unionOperators: boolean; } export const pythonOptions = { @@ -29,12 +33,38 @@ export const pythonOptions = { "python-version", "Python version", { - "3.5": { typeHints: false, dataClasses: false, builtinGenerics: false }, - "3.6": { typeHints: true, dataClasses: false, builtinGenerics: false }, - "3.7": { typeHints: true, dataClasses: true, builtinGenerics: false }, - "3.9": { typeHints: true, dataClasses: true, builtinGenerics: true }, - }, - "3.9", + "3.5": { + typeHints: false, + dataClasses: false, + builtinGenerics: false, + unionOperators: false, + }, + "3.6": { + typeHints: true, + dataClasses: false, + builtinGenerics: false, + unionOperators: false, + }, + "3.7": { + typeHints: true, + dataClasses: true, + builtinGenerics: false, + unionOperators: false, + }, + "3.9": { + typeHints: true, + dataClasses: true, + builtinGenerics: true, + unionOperators: false, + }, + "3.10": { + typeHints: true, + dataClasses: true, + builtinGenerics: true, + unionOperators: true, + }, + } satisfies Record, + "3.10", ), justTypes: new BooleanOption("just-types", "Classes only", false), nicePropertyNames: new BooleanOption( diff --git a/test/fixtures/python/run.sh b/test/fixtures/python/run.sh index 98c4273a0..011cad464 100755 --- a/test/fixtures/python/run.sh +++ b/test/fixtures/python/run.sh @@ -3,7 +3,7 @@ if [ "x$QUICKTYPE_PYTHON_VERSION" = "x2.7" ] ; then PYTHON="python2.7" else - PYTHON="python3.9" + PYTHON="python3.12" fi "$PYTHON" "$@" diff --git a/test/languages.ts b/test/languages.ts index 638c3a91d..79f7046ab 100644 --- a/test/languages.ts +++ b/test/languages.ts @@ -263,7 +263,13 @@ export const PythonLanguage: Language = { "keyword-unions.schema", // Requires more than 255 arguments ], rendererOptions: {}, - quickTestRendererOptions: [{ "python-version": "3.5" }], + quickTestRendererOptions: [ + // The default is "3.10"; keep the older feature sets covered. + { "python-version": "3.5" }, + { "python-version": "3.6" }, + { "python-version": "3.7" }, + { "python-version": "3.9" }, + ], sourceFiles: ["src/language/Python/index.ts"], }; From 96f3acca3c42fcab6684dfe428b1b726f5ea6020 Mon Sep 17 00:00:00 2001 From: Mark Probst Date: Sun, 19 Jul 2026 18:50:58 -0400 Subject: [PATCH 7/7] Give = None defaults to Any/None dataclass fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With dataclasses (3.7+), optional properties become nullable unions and get a " = None" default — except when the property type is Any (null is absorbed into it) or plain null (the union collapses), which got no default yet could sort after defaulted fields, making the generated dataclass raise "TypeError: non-default argument follows default argument" at import. This was latent on master because CI only ever exercised 3.5/3.6, which don't use dataclasses; with 3.10 as the new default it breaks e.g. vega-lite.schema and the combinations samples. Emit " = None" for root-level Any/None fields too, and make the property sort order compute exactly "will this field render with a default" via followTargetType — the old kind-based predicate saw pre-transformation types (transformed unions have kind "any" at sort time) and could order a non-defaulted field after defaulted ones. Co-Authored-By: Claude Fable 5 --- .../src/language/Python/PythonRenderer.ts | 78 +++++++++++-------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/packages/quicktype-core/src/language/Python/PythonRenderer.ts b/packages/quicktype-core/src/language/Python/PythonRenderer.ts index f21101c7b..e0d60263a 100644 --- a/packages/quicktype-core/src/language/Python/PythonRenderer.ts +++ b/packages/quicktype-core/src/language/Python/PythonRenderer.ts @@ -28,11 +28,7 @@ import { type Type, UnionType, } from "../../Type/index.js"; -import { - matchType, - nullableFromUnion, - removeNullFromUnion, -} from "../../Type/TypeUtils.js"; +import { matchType, removeNullFromUnion } from "../../Type/TypeUtils.js"; import { forbiddenPropertyNames, forbiddenTypeNames } from "./constants.js"; import type { pythonOptions } from "./language.js"; @@ -202,20 +198,44 @@ export class PythonRenderer extends ConvenienceRenderer { union.push(quote); + if (hasNull !== null) { + union.push(...this.noneDefault(isRootTypeDef)); + } + + return union; + } + + // A `" = None"` default for a class property whose value can be `None`. + // Only emitted for root level type defs, otherwise we may get type defs + // like `List[Optional[int] = None]`, which are invalid. Every property + // that gets a default must sort after all properties that don't — see + // `sortClassProperties`. + private noneDefault(isRootTypeDef: boolean): string[] { if ( - hasNull !== null && + isRootTypeDef && !this.getAlphabetizeProperties() && (this.pyOptions.features.dataClasses || - this.pyOptions.pydanticBaseModel) && - isRootTypeDef + this.pyOptions.pydanticBaseModel) ) { - // Only push "= None" if this is a root level type def - // otherwise we may get type defs like list[int | None = None] - // which are invalid - union.push(" = None"); + return [" = None"]; } - return union; + return []; + } + + // Does `pythonType(p.type, true)` end in a `" = None"` default? This + // must mirror the `noneDefault` calls in `pythonType` exactly: nullable + // unions, plus `Any` and `None` typed properties — an optional `Any` + // stays `Any` (`null` is absorbed by it), and an optional `null` + // collapses to just `null`, so those also default to `None`. + private classPropertyHasNoneDefault(p: ClassProperty): boolean { + const actualType = followTargetType(p.type); + if (actualType instanceof UnionType) { + const [hasNull] = removeNullFromUnion(actualType); + return hasNull !== null; + } + + return actualType.kind === "any" || actualType.kind === "null"; } protected pythonType( @@ -227,8 +247,11 @@ export class PythonRenderer extends ConvenienceRenderer { return matchType( actualType, - (_anyType) => this.withTyping("Any"), - (_nullType) => "None", + (_anyType) => [ + this.withTyping("Any"), + ...this.noneDefault(_isRootTypeDef), + ], + (_nullType) => ["None", ...this.noneDefault(_isRootTypeDef)], (_boolType) => "bool", (_integerType) => "int", (_doubletype) => "float", @@ -266,18 +289,7 @@ export class PythonRenderer extends ConvenienceRenderer { ); if (hasNull !== null) { - const rest: string[] = []; - if ( - !this.getAlphabetizeProperties() && - (this.pyOptions.features.dataClasses || - this.pyOptions.pydanticBaseModel) && - _isRootTypeDef - ) { - // Only push "= None" if this is a root level type def - // otherwise we may get type defs like List[Optional[int] = None] - // which are invalid - rest.push(" = None"); - } + const rest = this.noneDefault(_isRootTypeDef); if (nonNulls.size > 1) { this.withImport("typing", "Union"); @@ -412,13 +424,11 @@ export class PythonRenderer extends ConvenienceRenderer { this.pyOptions.features.dataClasses || this.pyOptions.pydanticBaseModel ) { - return mapSortBy(properties, (p: ClassProperty) => { - return (p.type instanceof UnionType && - nullableFromUnion(p.type) !== null) || - p.isOptional - ? 1 - : 0; - }); + // Properties that get a `" = None"` default must come after all + // properties that don't, or the generated dataclass is invalid. + return mapSortBy(properties, (p: ClassProperty) => + this.classPropertyHasNoneDefault(p) ? 1 : 0, + ); } return super.sortClassProperties(properties, propertyNames);