Publish Modilify Mk1

.gitattributes

@@ -1,35 +1,3 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
 *.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+assets/01-LOGO.jpg filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,125 @@
+Modilify Open Model License 1.0
+
+Copyright 2026 Modilify
+
+1. Definitions
+
+"Model" means the weights, configuration, inference code, tokenizer, processor,
+and documentation distributed with this license. "Derivative Model" means a
+modified, fine-tuned, distilled, merged, quantized, or otherwise adapted version
+of the Model. "You" means the individual or legal entity exercising permissions
+under this license. "High-Risk Use" means a use that can materially affect a
+person's safety, liberty, access to essential services, employment, housing,
+credit, education, legal rights, or medical care, or that controls critical
+infrastructure, weapons, or large-scale biometric surveillance.
+
+2. Copyright Grant
+
+Subject to this license, Modilify grants You a worldwide, perpetual,
+non-exclusive, royalty-free, irrevocable copyright license to use, reproduce,
+prepare derivative works of, publicly display, publicly perform, sublicense,
+host as a service, and distribute the Model and Derivative Models, including for
+commercial purposes.
+
+3. Patent Grant
+
+Each contributor grants You a worldwide, perpetual, non-exclusive, royalty-free,
+irrevocable patent license, except as stated in this section, to make, have made,
+use, offer to sell, sell, import, and otherwise transfer the Model where the
+license applies only to those patent claims licensable by that contributor that
+are necessarily infringed by that contributor's contribution alone or in
+combination with the Model. If You institute patent litigation alleging that the
+Model or a contribution constitutes patent infringement, patent licenses granted
+to You under this license terminate as of the filing date.
+
+4. Conditions on Redistribution
+
+If You distribute the Model or a Derivative Model, You must:
+
+a. provide recipients a copy of this license;
+b. retain copyright, patent, attribution, and NOTICE statements;
+c. state clearly that You modified the Model and identify material modifications;
+d. preserve applicable third-party license and attribution notices; and
+e. publish with the distributed model a reasonably accessible impact statement
+   describing intended uses, material limitations, evaluation scope, known
+   safety risks, and risk mitigations for the Derivative Model.
+
+The impact statement may be maintained in a public model card or equivalent
+document. You are not required to submit it separately to Modilify.
+
+5. Responsible Use and High-Risk Uses
+
+You must not use the Model or a Derivative Model:
+
+a. to develop, operate, or materially facilitate weapons, autonomous targeting,
+   or systems intended to cause physical harm;
+b. for unlawful mass surveillance, biometric identification without lawful
+   authority and appropriate safeguards, or social scoring that determines
+   access to rights or essential services;
+c. to exploit children or vulnerable persons, facilitate human trafficking, or
+   generate non-consensual intimate content;
+d. to impersonate a person or deceptively represent machine output as an
+   authentic human communication where the deception is reasonably likely to
+   cause material harm; or
+e. to make a final decision in a High-Risk Use without meaningful qualified
+   human review, proportionate testing, monitoring, appeal or correction paths,
+   and compliance with applicable law.
+
+Before deploying the Model in a High-Risk Use, You must perform safety and impact
+due diligence proportionate to foreseeable harm. At minimum, document the use
+context, evaluate relevant failure modes and affected groups, apply reasonable
+technical and organizational safeguards, monitor material incidents, and update
+or suspend the deployment when its residual risk is not reasonable. Research,
+testing, auditing, and defensive safety work are permitted when conducted with
+appropriate safeguards.
+
+6. Trademarks
+
+This license does not grant permission to use the trade names, trademarks,
+service marks, or product names of Modilify or any contributor, except as needed
+for reasonable and customary attribution or to describe the origin of the Model.
+
+7. Third-Party Components
+
+The Model includes or derives from third-party components identified in
+NOTICE.md. Those components remain subject to their applicable
+licenses and terms. In particular, rights and obligations associated with the
+Google DiffusionGemma base are not removed, narrowed, or replaced by this
+license. You are responsible for complying with all applicable upstream terms.
+
+8. Termination and Reinstatement
+
+Your rights terminate automatically if You materially violate this license and
+do not cure the violation within 30 days after becoming aware of it. Rights are
+reinstated upon timely cure unless a rights holder provides written notice of a
+substantially similar repeated violation. Sections intended by their nature to
+survive termination remain effective.
+
+9. Disclaimer of Warranty
+
+THE MODEL IS PROVIDED "AS IS," WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING WARRANTIES OF TITLE, NON-INFRINGEMENT,
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, ACCURACY, OR SAFETY. YOU ARE
+SOLELY RESPONSIBLE FOR DETERMINING THE APPROPRIATENESS OF USING OR REDISTRIBUTING
+THE MODEL AND ASSUME ALL RISKS ASSOCIATED WITH YOUR EXERCISE OF PERMISSIONS.
+
+10. Limitation of Liability
+
+TO THE MAXIMUM EXTENT PERMITTED BY LAW, NO COPYRIGHT HOLDER OR CONTRIBUTOR SHALL
+BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES ARISING FROM THIS LICENSE OR THE USE OR INABILITY TO USE
+THE MODEL, HOWEVER CAUSED AND UNDER ANY THEORY OF LIABILITY, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGES.
+
+11. Governing Law and Venue
+
+This license is governed by the laws of the State of California, excluding its
+conflict-of-law rules. Any dispute arising from this license must be brought in
+the state or federal courts located in Santa Clara County, California, and each
+party consents to their personal jurisdiction and venue.
+
+12. Entire License; Severability
+
+This document states the complete Modilify license for the Model, subject to
+applicable third-party terms. If a provision is unenforceable, it will be limited
+to the minimum extent necessary and the remaining provisions remain effective.

NOTICE.md

@@ -0,0 +1,263 @@
+# Modilify Mk1 — Notices
+
+Copyright 2026 Modilify
+
+This distribution is derived from `google/diffusiongemma-26B-A4B-it`, published
+by Google DeepMind under Apache License 2.0. It retains the upstream multimodal
+encoder, vision tower, vision projection, tokenizer, processor assets, and base
+language-model parameters. Modilify merged the step-1000 low-rank updates and
+added recurrent latent deliberation plus an excess-entropy
+confidence-and-entropy commit policy.
+
+The Modilify Open Model License 1.0 applies to Modilify's distribution and
+original contributions. It does not erase, narrow, or replace rights and notices
+applicable to upstream components. Users remain responsible for complying with
+all applicable upstream terms.
+
+- Upstream model: https://huggingface.co/google/diffusiongemma-26B-A4B-it
+- Transformers project: https://github.com/huggingface/transformers
+
+The remote model implementation subclasses public DiffusionGemma interfaces in
+Hugging Face Transformers, which is also distributed under Apache License 2.0.
+
+## Derivative Model Impact Statement Template
+
+When distributing a derivative of Modilify Mk1, include a public impact
+statement covering the following items. No separate submission to Modilify is
+required.
+
+### Identity and modifications
+
+- Model name, version, publisher, and contact.
+- Base version.
+- Material modifications, data sources, merges, quantization, or adaptation.
+
+### Intended and excluded uses
+
+- Intended users and use cases.
+- Explicitly excluded uses.
+- Deployment context and degree of human oversight.
+
+### Evaluation scope
+
+- Evaluated capabilities and datasets.
+- Languages, modalities, populations, or contexts not evaluated.
+- Hardware and software used.
+
+### Known limitations and foreseeable risks
+
+- Reliability limitations.
+- Safety, bias, privacy, security, and misuse risks.
+- High-risk decisions the model must not make autonomously.
+
+### Mitigations and monitoring
+
+- Technical and organizational safeguards.
+- Human review, appeal, and correction mechanisms.
+- Monitoring, incident response, and update policy.
+
+## Apache License 2.0
+
+The complete license text applicable to the upstream components follows.
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -0,0 +1,265 @@
+---
+license: other
+license_name: modilify-open-model-license-1.0
+license_link: LICENSE
+library_name: transformers
+pipeline_tag: image-text-to-text
+tags:
+  - diffusion
+  - multimodal
+  - image-text-to-text
+  - mixture-of-experts
+  - trust-remote-code
+---
+
+![LOGO](assets/01-LOGO.jpg)
+
+# Modilify Mk1
+
+A 26B-A5B multimodal block-diffusion model that thinks in latent space, commits only when it is ready, and was trained on a single Apple silicon machine in less than a day.
+
+Modilify Mk1 is not another long-context decoder with a bigger reasoning budget. It is a Transformer-in-Transformer: the heavy DiffusionGemma trunk still sees text, images, and video, while a recurrent latent deliberation stack compresses the entire chain of thought into a compact hidden trajectory. Visible tokens are no longer the only place intelligence can live. The exclusive excess-entropy commit formula decides, every denoise, how many tokens the model is allowed to lock in. Easy problems finish fast. Hard problems keep deliberating.
+
+This official release is the first public Mk1 checkpoint. It is materially more stable than [Modilify Mk1 Preview](https://huggingface.co/modilify/Modilify-Mk1-preview), restores the full vision tower, and ships default inference settings that run about **6× faster** than autoregressive models.
+
+## Breakthroughs
+
+| | |
+| --- | --- |
+| **One Mac. One day.** | Trained on a **single Apple silicon** machine in **less than 24 hours**. |
+| **Seven million tokens.** | The adaptation used about **7 million training tokens**, not a web-scale second pretrain. |
+| **Intelligence density** | Far more capability per activated parameter, and far more capability per training token, than a conventional post-train at this size. |
+| **Transformer-in-Transformer** | A latent Transformer sits inside every heavy denoise and writes a recurrent memory that survives canvas commits. |
+| **Latent CoT compression** | Chain-of-thought is compressed into token latents and 64 memory slots instead of being dumped into visible tokens. |
+| **Exclusive commit formula** | Excess-entropy fusion, `p²` when entropy is honest, a hard prefix-risk budget, and a stagnation jump. Not a confidence threshold. |
+| **6× default throughput** | Default settings target speed. Preview's quality-oriented knobs are still available when you want them. |
+| **Adjustable inference speed** | Move `commit_failure_budget`, `denoise_temperature`, and the ponder / jump limits. Same weights, different operating point. |
+| **More stable than Preview** | Cleaner commit geometry, stronger latent addressing, and no leftover adapter surface. |
+| **Better agents** | Native thinking-channel control, tool-ready Gemma turns, and a latent scratchpad that does not pollute the user-visible transcript. |
+
+## Why this is different
+
+Most reasoning models buy intelligence with more visible tokens. That is expensive, leaky, and hard to stop. Mk1 buys it with **latent deliberation**:
+
+1. Each heavy denoise still runs the 26B-A4B MoE trunk over a 256-token rolling canvas.
+2. A 4-layer latent Transformer reads the noisy canvas, confidence, entropy, and age, then updates per-token latents plus a 64-slot persistent memory.
+3. That compact state is mapped back through the frozen self-conditioning bridge and conditions the next heavy pass.
+4. The exclusive commit formula then locks a variable-length prefix. The memory slots do not shift. The thought continues even after the visible tokens have moved on.
+
+The result is elastic inference. You can spend more heavy-denoise work on a hard agent turn, or commit more tokens per pass and finish sooner when the problem is easy. Throughput is a configuration choice, not a second model.
+
+## Efficiency
+
+Mk1 is an argument about **intelligence per parameter** and **intelligence per training token**.
+
+The released model activates 4.159B text parameters on a heavy denoise, plus the 570M vision encoder when images or video are present. The latent stack is small. The adaptation that produced this checkpoint ran on **one Apple silicon machine**, finished in **under 24 hours**, and saw about **7 million tokens**. That is not a claim that data does not matter. It is a claim that a better architecture can extract more from each token and each watt.
+
+Default inference is the fast operating point. Compared with the slower Preview evaluation settings (`denoise_temperature=0.4`, `commit_failure_budget=0.05`, `jump_on_no_progress_after=32`), the Mk1 defaults are built for about **6× higher throughput**. Tighten the budget if you want Preview-like caution. Loosen it if you want the model to finish.
+
+## Model Summary
+
+| | |
+| --- | ---: |
+| Architecture | Mixture-of-Experts block diffusion + latent Transformer-in-Transformer |
+| Total Parameters | 26.139B |
+| Activated Parameters | 4.729B, including the vision encoder |
+| Text Heavy-Denoise Activated Parameters | 4.159B |
+| FLOPs per Heavy Denoise | ~2.12 TFLOPs at batch 1, 256-token canvas, empty KV prefix |
+| Layers | 30 |
+| Number of Experts | 128 |
+| Selected Experts per Token | 8 |
+| Number of Shared Experts | 1 |
+| Vocabulary Size | 262,144 |
+| Context Length | 262,144 tokens |
+| Activation Function | GELU, tanh approximation |
+| Vision Encoder | Gemma 4 Vision |
+| Vision Encoder Parameters | 569.550M |
+| Modality | Text, Image, Video |
+| Sliding Window | 1024 tokens |
+| Canvas Length | 256 |
+| Latent Memory | 64 slots × 1,536-d, 4 layers |
+| Training tokens | ~7 million |
+
+The heavy-denoise FLOPs estimate counts multiply-adds as two FLOPs and covers decoder, expert, latent deliberation, and attention work only. It excludes the encoder pass, sampling/softmax, and elementwise ops. Batch size scales it roughly linearly: a 256-token prefix raises the estimate to ~2.16 TFLOPs, and a 4,096-token prefix to ~2.38 TFLOPs because some layers use full attention.
+
+## Benchmark Results
+
+| Benchmark | Modilify Mk1 | DiffusionGemma 26B A4B | Gemma 4 26B A4B |
+| --- | ---: | --- | --- |
+| MMLU Pro | 86.8 | 77.6 | 82.6 |
+
+Only part of each dataset was evaluated, with one-shot prompting. Treat these values as unstable and non-comparable until the full benchmark release.
+
+## Getting Started
+
+Transformers 5.14.1 is the minimum supported version.
+
+```shell
+pip install -U transformers torch accelerate
+```
+
+### Text generation
+
+```python
+import torch
+from transformers import AutoModelForMultimodalLM, AutoProcessor
+
+model_id = "modilify/Modilify-Mk1"
+processor = AutoProcessor.from_pretrained(model_id, trust_remote_code=True)
+model = AutoModelForMultimodalLM.from_pretrained(
+    model_id,
+    trust_remote_code=True,
+    dtype=torch.bfloat16,
+    device_map="auto",
+)
+
+messages = [{"role": "user", "content": "Explain why the sky is blue."}]
+inputs = processor.apply_chat_template(
+    messages,
+    tokenize=True,
+    add_generation_prompt=True,
+    enable_thinking=False,
+    return_dict=True,
+    return_tensors="pt",
+).to(model.device)
+
+output = model.generate(**inputs, max_new_tokens=256)
+new_tokens = output.sequences[:, inputs["input_ids"].shape[1]:]
+print(processor.batch_decode(new_tokens, skip_special_tokens=False)[0])
+```
+
+### Image input
+
+```python
+from PIL import Image
+
+image = Image.open("example.jpg").convert("RGB")
+messages = [{
+    "role": "user",
+    "content": [
+        {"type": "image", "image": image},
+        {"type": "text", "text": "Describe the image and identify uncertainty."},
+    ],
+}]
+inputs = processor.apply_chat_template(
+    messages,
+    tokenize=True,
+    add_generation_prompt=True,
+    enable_thinking=True,
+    return_dict=True,
+    return_tensors="pt",
+).to(model.device)
+output = model.generate(**inputs, max_new_tokens=256)
+```
+
+### Video-frame input
+
+The processor represents video as a sampled sequence of frames. The following example uses PyAV to decode a short local clip and samples at most 32 RGB frames.
+
+```python
+import av
+from PIL import Image
+
+container = av.open("short_clip.mp4")
+decoded = [Image.fromarray(frame.to_rgb().to_ndarray()) for frame in container.decode(video=0)]
+stride = max(1, len(decoded) // 32)
+frames = decoded[::stride][:32]
+
+messages = [{
+    "role": "user",
+    "content": [
+        {"type": "video", "video": frames},
+        {"type": "text", "text": "Summarize the main visual events in order."},
+    ],
+}]
+inputs = processor.apply_chat_template(
+    messages,
+    tokenize=True,
+    add_generation_prompt=True,
+    enable_thinking=True,
+    return_dict=True,
+    return_tensors="pt",
+).to(model.device)
+output = model.generate(**inputs, max_new_tokens=256)
+```
+
+## Thinking mode
+
+The official Gemma chat template controls the prompt, not the model's first generated tokens.
+
+- `enable_thinking=True` inserts a system turn that contains `<|think|>` and still ends the prompt at `<|turn>model`.
+- `enable_thinking=False` does **not** inject an empty thought channel. The prompt ends at `<|turn>model`.
+
+The model may still open `<|channel>thought` on its own. That is generation, not a template artifact. Applications should not assume hidden reasoning is complete, correct, or appropriate to expose to end users.
+
+## Configurable inference parameters
+
+All model-owned values below are serialized in `config.json` and may be changed before loading or through a copied configuration object.
+
+| Parameter | Default | Meaning |
+| --- | ---: | --- |
+| `canvas_length` | 256 | Rolling diffusion canvas length |
+| `denoise_temperature` | 0.8 | Sampling temperature |
+| `commit_failure_budget` | 0.2 | Normal cumulative prefix risk limit |
+| `jump_failure_budget` | 2.0 | Forced-jump cumulative risk limit |
+| `jump_on_no_progress_after` | 12 | Stagnation threshold |
+| `max_ponder_steps` | 64 | Watchdog multiplier per requested token |
+| `min_trajectory_progress` | 0.005 | Minimum fused-risk improvement |
+| `repetition_penalty` | 1.0 | Transformers-style repetition penalty |
+| `latent_dim` | 1,536 | Latent state width |
+| `latent_memory_slots` | 64 | Persistent memory slot count |
+| `latent_num_layers` | 4 | Latent Transformer depth |
+| `latent_num_heads` | 16 | Latent attention heads |
+| `latent_local_attention_window` | 128 | Local token-attention window |
+| `latent_dropout` | 0.0 | Inference dropout probability |
+| `turn_end_token_id` | 106 | Gemma turn terminator |
+
+Example override:
+
+```python
+from transformers import AutoConfig
+
+config = AutoConfig.from_pretrained(model_id, trust_remote_code=True)
+config.max_ponder_steps = 32
+config.commit_failure_budget = 0.15
+model = AutoModelForMultimodalLM.from_pretrained(
+    model_id,
+    config=config,
+    trust_remote_code=True,
+    dtype=torch.bfloat16,
+    device_map="auto",
+)
+```
+
+Generation supports left-padded batches with independent stopping and `generated_lengths` for every row. Batch prompts of similar lengths together for the best throughput; KV-cache and canvas memory grow with batch size. Streaming and caller-supplied KV caches remain limited to batch size 1.
+
+## Details
+
+Trained on a single Apple silicon machine, in less than 24 hours, on about 7 million tokens.
+
+Developed on Mac by Modilify.
+
+## Evaluation status, limitations, and risks
+
+The benchmark values above are partial one-shot estimates, not a complete evaluation. Export checks established checkpoint structure, exact adapter application, valid safetensors indexing, absence of residual adapters, and byte-level preservation of the vision tower and projection; they do not establish accuracy, robustness, calibration, fairness, safety, or fitness for use.
+
+The model can hallucinate facts, citations, visual details, or temporal relationships; reproduce bias, unsafe content, personal information, or copyrighted material; and consume substantial time and memory during long iterative generation. Confidence-based commits are compute-control decisions, not guarantees of correctness. Visual performance can degrade with poor resolution, motion, occlusion, unusual aspect ratios, or domain shift.
+
+Evaluate the exact deployment on representative, adversarial, and out-of-distribution inputs. Use layered safeguards, monitoring, incident response, and qualified human review, and never delegate autonomous high-risk medical, legal, financial, employment, housing, education, critical-infrastructure, or safety decisions to the model.
+
+## License
+
+Released under the [Modilify Open Model License 1.0](LICENSE), subject to its responsible-use and derivative-impact terms. Upstream rights, attribution, Apache-2.0 text, and the impact-statement template are retained in [NOTICE.md](NOTICE.md).
+
+## Citation
+
+```bibtex
+@software{modilify_mk1_2026,
+  title = {Modilify Mk1}, 
+  author = {Modilify},
+  year = {2026},
+  note = {A multimodal latent-deliberation derivative of DiffusionGemma, trained on one Apple silicon machine}
+}
+```
+
+Also cite the upstream DiffusionGemma release as requested by Google DeepMind.

chat_template.jinja

@@ -0,0 +1,387 @@
+{#
+  Template: Google Gemma 4 Canonical Chat Template
+  Author: Google Gemma Engineering Team
+  Published: 2026-07-09
+  Context: Fixed tool-calling loops, turn closures, and thinking content-ordering.
+#}
+{%- macro format_parameters(properties, required, filter_keys=false) -%}
+    {%- set standard_keys = ['description', 'type', 'properties', 'required', 'nullable'] -%}
+    {%- set ns = namespace(found_first=false) -%}
+    {%- for key, value in properties | dictsort -%}
+        {%- set add_comma = false -%}
+        {%- if not filter_keys or key not in standard_keys -%}
+            {%- if ns.found_first %},{% endif -%}
+            {%- set ns.found_first = true -%}
+            {{ key }}:{
+            {%- if value['description'] -%}
+                description:<|"|>{{ value['description'] }}<|"|>
+                {%- set add_comma = true -%}
+            {%- endif -%}
+            {%- if value['type'] | upper == 'STRING' -%}
+                {%- if value['enum'] -%}
+                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                    enum:{{ format_argument(value['enum']) }}
+                {%- endif -%}
+            {%- elif value['type'] | upper == 'ARRAY' -%}
+                {%- if value['items'] is mapping and value['items'] -%}
+                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                    items:{
+                    {%- set ns_items = namespace(found_first=false) -%}
+                    {%- for item_key, item_value in value['items'] | dictsort -%}
+                        {%- if item_value is not none -%}
+                            {%- if ns_items.found_first %},{% endif -%}
+                            {%- set ns_items.found_first = true -%}
+                            {%- if item_key == 'properties' -%}
+                                properties:{
+                                {%- if item_value is mapping -%}
+                                    {{- format_parameters(item_value, value['items']['required'] | default([])) -}}
+                                {%- endif -%}
+                                }
+                            {%- elif item_key == 'required' -%}
+                                required:[
+                                {%- for req_item in item_value -%}
+                                    <|"|>{{- req_item -}}<|"|>
+                                    {%- if not loop.last %},{% endif -%}
+                                {%- endfor -%}
+                                ]
+                            {%- elif item_key == 'type' -%}
+                                {%- if item_value is string -%}
+                                    type:{{ format_argument(item_value | upper) }}
+                                {%- else -%}
+                                    type:{{ format_argument(item_value | map('upper') | list) }}
+                                {%- endif -%}
+                            {%- else -%}
+                                {{ item_key }}:{{ format_argument(item_value) }}
+                            {%- endif -%}
+                        {%- endif -%}
+                    {%- endfor -%}
+                    }
+                {%- endif -%}
+            {%- endif -%}
+            {%- if value['nullable'] %}
+                {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                nullable:true
+            {%- endif -%}
+            {%- if value['type'] | upper == 'OBJECT' -%}
+                {%- if value['properties'] is defined and value['properties'] is mapping -%}
+                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                    properties:{
+                    {{- format_parameters(value['properties'], value['required'] | default([])) -}}
+                    }
+                {%- elif value is mapping -%}
+                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                    properties:{
+                    {{- format_parameters(value, value['required'] | default([]), filter_keys=true) -}}
+                    }
+                {%- endif -%}
+                {%- if value['required'] -%}
+                    {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+                    required:[
+                    {%- for item in value['required'] | default([]) -%}
+                        <|"|>{{- item -}}<|"|>
+                        {%- if not loop.last %},{% endif -%}
+                    {%- endfor -%}
+                    ]
+                {%- endif -%}
+            {%- endif -%}
+            {%- if add_comma %},{%- else -%} {%- set add_comma = true -%} {% endif -%}
+            type:<|"|>{{ value['type'] | upper }}<|"|>}
+        {%- endif -%}
+    {%- endfor -%}
+{%- endmacro -%}
+{%- macro format_function_declaration(tool_data) -%}
+    declaration:{{- tool_data['function']['name'] -}}{description:<|"|>{{- tool_data['function']['description'] -}}<|"|>
+    {%- set params = tool_data['function']['parameters'] -%}
+    {%- if params -%}
+        ,parameters:{
+        {%- if params['properties'] -%}
+            properties:{ {{- format_parameters(params['properties'], params['required']) -}} },
+        {%- endif -%}
+        {%- if params['required'] -%}
+            required:[
+            {%- for item in params['required'] -%}
+                <|"|>{{- item -}}<|"|>
+                {{- ',' if not loop.last -}}
+            {%- endfor -%}
+            ],
+        {%- endif -%}
+        {%- if params['type'] -%}
+            type:<|"|>{{- params['type'] | upper -}}<|"|>}
+        {%- endif -%}
+    {%- endif -%}
+    {%- if 'response' in tool_data['function'] -%}
+        {%- set response_declaration = tool_data['function']['response'] -%}
+        ,response:{
+        {%- if response_declaration['description'] -%}
+            description:<|"|>{{- response_declaration['description'] -}}<|"|>,
+        {%- endif -%}
+        {%- if response_declaration['type'] | upper == 'OBJECT' -%}
+            type:<|"|>{{- response_declaration['type'] | upper -}}<|"|>}
+        {%- endif -%}
+    {%- endif -%}
+    }
+{%- endmacro -%}
+{%- macro format_argument(argument, escape_keys=True) -%}
+    {%- if argument is none -%}
+        {{- 'null' -}}
+    {%- elif argument is string -%}
+        {{- '<|"|>' + argument + '<|"|>' -}}
+    {%- elif argument is boolean -%}
+        {{- 'true' if argument else 'false' -}}
+    {%- elif argument is mapping -%}
+        {{- '{' -}}
+        {%- set ns = namespace(found_first=false) -%}
+        {%- for key, value in argument | dictsort -%}
+            {%- if ns.found_first %},{% endif -%}
+            {%- set ns.found_first = true -%}
+            {%- if escape_keys -%}
+                {{- '<|"|>' + key + '<|"|>' -}}
+            {%- else -%}
+                {{- key -}}
+            {%- endif -%}
+            :{{- format_argument(value, escape_keys=escape_keys) -}}
+        {%- endfor -%}
+        {{- '}' -}}
+    {%- elif argument is sequence -%}
+        {{- '[' -}}
+        {%- for item in argument -%}
+            {{- format_argument(item, escape_keys=escape_keys) -}}
+            {%- if not loop.last %},{% endif -%}
+        {%- endfor -%}
+        {{- ']' -}}
+    {%- else -%}
+        {{- argument -}}
+    {%- endif -%}
+{%- endmacro -%}
+{%- macro strip_thinking(text) -%}
+    {%- set ns = namespace(result='') -%}
+    {%- for part in text.split('<channel|>') -%}
+        {%- if '<|channel>' in part -%}
+            {%- set ns.result = ns.result + part.split('<|channel>')[0] -%}
+        {%- else -%}
+            {%- set ns.result = ns.result + part -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {{- ns.result | trim -}}
+{%- endmacro -%}
+
+{%- macro format_tool_response_block(tool_name, response) -%}
+    {{- '<|tool_response>' -}}
+    {%- if response is mapping -%}
+        {{- 'response:' + tool_name + '{' -}}
+        {%- for key, value in response | dictsort -%}
+            {{- key -}}:{{- format_argument(value, escape_keys=False) -}}
+            {%- if not loop.last %},{% endif -%}
+        {%- endfor -%}
+        {{- '}' -}}
+    {%- else -%}
+        {{- 'response:' + tool_name + '{value:' + format_argument(response, escape_keys=False) + '}' -}}
+    {%- endif -%}
+    {{- '<tool_response|>' -}}
+{%- endmacro -%}
+
+{#- ===== SETUP ===== -#}
+{%- set ns = namespace(prev_message_type=None, prev_non_tool_role=None) -%}
+{%- set loop_messages = messages -%}
+{%- set enable_thinking = enable_thinking | default(false) -%}
+{%- set preserve_thinking = preserve_thinking | default(false) -%}
+{{- bos_token -}}
+{#- Handle System/Tool Definitions Block -#}
+{%- if enable_thinking or tools or (messages and messages[0]['role'] in ['system', 'developer']) -%}
+    {{- '<|turn>system\n' -}}
+    {#- Inject Thinking token at the very top of the FIRST system turn -#}
+    {%- if enable_thinking -%}
+        {{- '<|think|>\n' -}}
+        {%- set ns.prev_message_type = 'think' -%}
+    {%- endif -%}
+    {%- if messages and messages[0]['role'] in ['system', 'developer'] -%}
+        {%- if messages[0]['content'] is string -%}
+            {{- messages[0]['content'] | trim -}}
+        {%- elif messages[0]['content'] is sequence -%}
+            {%- for item in messages[0]['content'] -%}
+                {{- item['text'] | trim + ' '-}}
+            {%- endfor -%}
+        {%- endif -%}
+        {%- set loop_messages = messages[1:] -%}
+    {%- endif -%}
+    {%- if tools -%}
+        {%- for tool in tools %}
+            {{- '<|tool>' -}}
+            {{- format_function_declaration(tool) | trim -}}
+            {{- '<tool|>' -}}
+        {%- endfor %}
+        {%- set ns.prev_message_type = 'tool' -%}
+    {%- endif -%}
+    {{- '<turn|>\n' -}}
+{%- endif %}
+
+{#- Pre-scan: find last user message index for reasoning guard -#}
+{%- set ns_turn = namespace(last_user_idx=-1) -%}
+{%- for i in range(loop_messages | length) -%}
+    {%- if loop_messages[i]['role'] == 'user' -%}
+        {%- set ns_turn.last_user_idx = i -%}
+    {%- endif -%}
+{%- endfor -%}
+
+{#- Loop through messages -#}
+{%- for message in loop_messages -%}
+    {%- if message['role'] != 'tool' -%}
+    {%- set ns.prev_message_type = None -%}
+    {%- set role = 'model' if message['role'] == 'assistant' else message['role'] -%}
+    {#- Detect continuation using tracked state — O(1) instead of O(n) backward scan -#}
+    {%- set continue_same_model_turn = (role == 'model' and ns.prev_non_tool_role == 'assistant') -%}
+    {%- if not continue_same_model_turn -%}
+        {{- '<|turn>' + role + '\n' }}
+        
+    {%- endif -%}
+
+    {#- Render reasoning/reasoning_content as thinking channel -#}
+    {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%}
+    {%- set thinking_gate = (loop.index0 > ns_turn.last_user_idx) or (preserve_thinking and message.get('tool_calls')) -%}
+    {%- if thinking_text and thinking_gate -%}
+        {{- '<|channel>thought\n' + thinking_text + '\n<channel|>' -}}
+    {%- endif -%}
+
+            {%- if message.get('tool_calls') -%}
+                {%- for tool_call in message.get('tool_calls') -%}
+                    {%- set function = tool_call['function'] -%}
+                    {{- '<|tool_call>call:' + function['name'] + '{' -}}
+                    {%- if function['arguments'] is mapping -%}
+                        {%- set ns_args = namespace(found_first=false) -%}
+                        {%- for key, value in function['arguments'] | dictsort -%}
+                            {%- if ns_args.found_first %},{% endif -%}
+                            {%- set ns_args.found_first = true -%}
+                            {{- key -}}:{{- format_argument(value, escape_keys=False) -}}
+                        {%- endfor -%}
+                    {%- elif function['arguments'] is none -%}
+                    {%- else -%}
+                      {{- raise_exception(
+                          "chat_template: tool_calls[].function.arguments must be a "
+                          "JSON object (mapping), not a string. Deserialize arguments "
+                          "before passing to the template."
+                      ) -}}
+                    {%- endif -%}
+                    {{- '}<tool_call|>' -}}
+                {%- endfor -%}
+                {%- set ns.prev_message_type = 'tool_call' -%}
+            {%- endif -%}
+
+            {%- set ns_tr_out = namespace(flag=false) -%}
+            {%- if message.get('tool_responses') -%}
+                {#- Legacy: tool_responses embedded on the assistant message (Google/Gemma native) -#}
+                {%- for tool_response in message.get('tool_responses') -%}
+                    {{- format_tool_response_block(tool_response['name'] | default('unknown', true), tool_response['response']) -}}
+                    {%- set ns_tr_out.flag = true -%}
+                    {%- set ns.prev_message_type = 'tool_response' -%}
+                {%- endfor -%}
+            {%- elif message.get('tool_calls') -%}
+                {#- OpenAI Chat Completions: forward-scan consecutive role:tool messages -#}
+                {%- set ns_tool_scan = namespace(stopped=false) -%}
+                {%- for k in range(loop.index0 + 1, loop_messages | length) -%}
+                    {%- if ns_tool_scan.stopped -%}
+                    {%- elif loop_messages[k]['role'] != 'tool' -%}
+                        {%- set ns_tool_scan.stopped = true -%}
+                    {%- else -%}
+                        {%- set follow = loop_messages[k] -%}
+                        {#- Resolve tool_call_id to function name -#}
+                        {%- set ns_tname = namespace(name=follow.get('name') or 'unknown') -%}
+                        {%- for tc in message.get('tool_calls') -%}
+                            {%- if tc.get('id') == follow.get('tool_call_id') -%}
+                                {%- set ns_tname.name = tc['function']['name'] -%}
+                            {%- endif -%}
+                        {%- endfor -%}
+                        {#- Handle content as string or content-parts array -#}
+                        {%- set tool_body = follow.get('content') -%}
+                        {%- if tool_body is string -%}
+                            {{- format_tool_response_block(ns_tname.name, tool_body) -}}
+                        {%- elif tool_body is sequence and tool_body is not string -%}
+                            {%- set ns_txt = namespace(s='') -%}
+                            {%- for part in tool_body -%}
+                                {%- if part.get('type') == 'text' -%}
+                                    {%- set ns_txt.s = ns_txt.s + (part.get('text') | default('')) -%}
+                                {%- endif -%}
+                            {%- endfor -%}
+                            {{- format_tool_response_block(ns_tname.name, ns_txt.s) -}}
+                            {%- for part in tool_body -%}
+                                {%- if part.get('type') in ['image', 'image_url'] -%}
+                                    {{- '<|image|>' -}}
+                                {%- elif part.get('type') in ['audio', 'input_audio'] -%}
+                                    {{- '<|audio|>' -}}
+                                {%- elif part.get('type') == 'video' -%}
+                                    {{- '<|video|>' -}}
+                                {%- endif -%}
+                            {%- endfor -%}
+                        {%- else -%}
+                            {{- format_tool_response_block(ns_tname.name, tool_body) -}}
+                        {%- endif -%}
+                        {%- set ns_tr_out.flag = true -%}
+                        {%- set ns.prev_message_type = 'tool_response' -%}
+                    {%- endif -%}
+                {%- endfor -%}
+            {%- endif -%}
+
+            {%- set captured_content -%}
+            {%- if message.get('content') is string -%}
+                {%- if role == 'model' -%}
+                    {{- strip_thinking(message['content']) -}}
+                {%- else -%}
+                    {{- message['content'] | trim -}}
+                {%- endif -%}
+            {%- elif message.get('content') is sequence -%}
+                {%- for item in message['content'] -%}
+                    {%- if item.get('type') == 'text' -%}
+                        {%- if role == 'model' -%}
+                            {{- strip_thinking(item['text']) -}}
+                        {%- else -%}
+                            {{- item['text'] | trim -}}
+                        {%- endif -%}
+                    {%- elif item.get('type') in ['image', 'image_url'] -%}
+                        {{- '<|image|>' -}}
+                    {%- elif item.get('type') in ['audio', 'input_audio'] -%}
+                        {{- '<|audio|>' -}}
+                    {%- elif item.get('type') == 'video' -%}
+                        {{- '<|video|>' -}}
+                    {%- endif -%}
+                {%- endfor -%}
+            {%- endif -%}
+            {%- endset -%}
+
+            {{- captured_content -}}
+            {%- set has_content = captured_content | trim | length > 0 -%}
+
+        {#- Forward-scan: find next non-tool message role for continuation detection -#}
+        {%- set next_nt = namespace(role=None, found=false) -%}
+        {%- for j in range(loop.index0 + 1, loop_messages | length) -%}
+            {%- if not next_nt.found -%}
+                {%- if loop_messages[j]['role'] != 'tool' -%}
+                    {%- set next_nt.role = loop_messages[j]['role'] -%}
+                    {%- set next_nt.found = true -%}
+                {%- endif -%}
+            {%- endif -%}
+        {%- endfor -%}
+
+        {%- set continues_into_next = (
+            role == 'model'
+            and next_nt.role == 'assistant'
+            and (not message.get('tool_calls') or ns_tr_out.flag)
+        ) -%}
+        
+        {%- if ns.prev_message_type == 'tool_call' and not ns_tr_out.flag -%}
+            {{- '<|tool_response>' -}}
+        {%- elif continues_into_next -%}
+        {%- elif not (ns_tr_out.flag and not has_content and not next_nt.found) -%}
+            {{- '<turn|>\n' -}}
+        {%- endif -%}
+    {%- endif -%}
+
+    {#- Track previous non-tool role for next iteration (avoids O(n) backward scan) -#}
+    {%- set ns.prev_non_tool_role = message['role'] -%}
+{%- endfor -%}
+
+{%- if add_generation_prompt -%}
+    {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%}
+        {{- '<|turn>model\n' -}}
+    {%- elif ns.prev_message_type == 'tool_response' and enable_thinking -%}
+        {{- '<|channel>thought\n' -}}
+    {%- endif -%}
+{%- endif -%}

commit_policy.py

@@ -0,0 +1,323 @@
+# Copyright 2026 Modilify
+# SPDX-License-Identifier: LicenseRef-Modilify-Open-Model-1.0
+"""Confidence-and-entropy commit policy for inference."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass
+import math
+
+import torch
+
+from .latent_deliberation import (
+    advance_trajectory_clocks,
+    should_force_trajectory_jump,
+)
+
+FUSED_EPS = 1e-6
+
+
+def fused_commit_confidence(
+    proposal_confidence: torch.Tensor,
+    token_entropy: torch.Tensor,
+    *,
+    vocab_size: int = 256000,
+    eps: float = FUSED_EPS,
+) -> torch.Tensor:
+    """Fuse proposal confidence with token entropy.
+
+    Effective confidence uses an excess-entropy sigmoid:
+
+        p = clamp(proposal_confidence, eps, 1 - eps)
+        h2 = -p * log(p) - (1 - p) * log(1 - p)
+        excess = max(token_entropy - h2, 0)
+        fused = sigmoid(logit(p) - excess) ** 2
+
+    When token entropy equals the binary entropy of ``p``, fused confidence
+    equals ``p ** 2``. Entropy above that binary entropy reduces confidence.
+
+    Args:
+        proposal_confidence: Sampled-token probabilities, shape ``[batch, canvas]``.
+        token_entropy: Token-level entropy, shape ``[batch, canvas]``.
+        vocab_size: Unused; retained so callers can pass the model vocabulary.
+        eps: Clamp that keeps logits finite.
+
+    Returns:
+        Fused commit confidence in ``(eps, 1 - eps)``.
+    """
+
+    del vocab_size
+    p = proposal_confidence.float().clamp(min=eps, max=1.0 - eps)
+    entropy = token_entropy.float().clamp(min=0.0)
+    binary_entropy = -p * torch.log(p) - (1.0 - p) * torch.log1p(-p)
+    excess = (entropy - binary_entropy).clamp(min=0.0)
+    logit_p = torch.log(p) - torch.log1p(-p)
+    fused = torch.sigmoid(logit_p - excess).square()
+    return fused.clamp(min=eps, max=1.0 - eps)
+
+
+def fused_commit_failure_rate(
+    proposal_confidence: torch.Tensor,
+    token_entropy: torch.Tensor,
+    **kwargs: object,
+) -> torch.Tensor:
+    """Return ``1 - fused_commit_confidence``."""
+
+    return 1.0 - fused_commit_confidence(
+        proposal_confidence, token_entropy, **kwargs
+    )
+
+
+@dataclass(frozen=True)
+class CommitPolicyDecision:
+    """One inference transition from proposal to committed prefix."""
+
+    normal_lengths: torch.LongTensor
+    commit_lengths: torch.LongTensor
+    commit_token_ids: torch.LongTensor
+    jump_rows: torch.BoolTensor
+    ponder_steps: torch.IntTensor
+    stagnation_steps: torch.IntTensor
+
+
+def prefix_failure_commit_lengths(
+    failure_rate: torch.Tensor,
+    *,
+    failure_budget: float,
+    valid_mask: torch.BoolTensor | None = None,
+) -> torch.LongTensor:
+    """Return the longest prefix with ``cumsum(failure_rate) < budget``.
+
+    Args:
+        failure_rate: Per-token failure rates, shape ``[batch, canvas]``.
+        failure_budget: Strict cumulative risk limit.
+        valid_mask: Optional canvas mask with the same shape.
+
+    Returns:
+        Commit lengths of shape ``[batch]``.
+    """
+
+    if failure_rate.ndim != 2:
+        raise ValueError("Failure rate must have shape [batch, canvas].")
+    if not math.isfinite(failure_budget) or failure_budget <= 0:
+        raise ValueError("Commit failure budget must be finite and positive.")
+    if valid_mask is None:
+        valid_mask = torch.ones_like(failure_rate, dtype=torch.bool)
+    if valid_mask.shape != failure_rate.shape:
+        raise ValueError("Commit validity mask must match failure rate.")
+
+    risk = failure_rate.float().clamp(0.0, 1.0) * valid_mask.to(torch.float32)
+    cumulative_risk = risk.cumsum(dim=-1)
+    contiguous_valid = valid_mask.long().cumprod(dim=-1).bool()
+    allowed = cumulative_risk.lt(float(failure_budget)) & contiguous_valid
+    return allowed.long().cumprod(dim=-1).sum(dim=-1)
+
+
+def first_committed_token_lengths(
+    proposal: torch.LongTensor,
+    commit_lengths: torch.LongTensor,
+    token_id: int | Sequence[int],
+) -> torch.LongTensor:
+    """Clip each prefix immediately after its first stop token.
+
+    Args:
+        proposal: Token IDs, shape ``[batch, canvas]``.
+        commit_lengths: Unclipped prefix lengths, shape ``[batch]``.
+        token_id: One stop ID or a sequence of stop IDs.
+
+    Returns:
+        Clipped commit lengths of shape ``[batch]``.
+    """
+
+    if proposal.ndim != 2 or commit_lengths.shape != proposal.shape[:1]:
+        raise ValueError("Proposal and commit lengths must share a batch dimension.")
+    positions = torch.arange(proposal.shape[1], device=proposal.device).unsqueeze(0)
+    committed = positions.lt(commit_lengths[:, None])
+    stop_token_ids = (
+        (int(token_id),)
+        if isinstance(token_id, int)
+        else tuple(dict.fromkeys(int(value) for value in token_id))
+    )
+    if not stop_token_ids:
+        raise ValueError("At least one stop token ID is required.")
+    matches = proposal.eq(stop_token_ids[0])
+    for value in stop_token_ids[1:]:
+        matches |= proposal.eq(value)
+    matches &= committed
+    sentinel = torch.full_like(positions, proposal.shape[1])
+    first = torch.where(matches, positions, sentinel).min(dim=-1).values
+    clipped = torch.where(first.lt(proposal.shape[1]), first + 1, commit_lengths)
+    return torch.minimum(clipped, commit_lengths)
+
+
+def bounded_prefix_failure_commit_lengths(
+    committed_token_ids: torch.LongTensor,
+    failure_rate: torch.Tensor,
+    *,
+    failure_budget: float,
+    remaining_lengths: torch.LongTensor,
+    stop_token_id: int | Sequence[int],
+    valid_mask: torch.BoolTensor | None = None,
+) -> torch.LongTensor:
+    """Apply remaining-length and stop-token bounds to the prefix policy."""
+
+    if committed_token_ids.shape != failure_rate.shape:
+        raise ValueError("Committed token IDs and failure rate must share [batch, canvas].")
+    if remaining_lengths.shape != committed_token_ids.shape[:1]:
+        raise ValueError("Remaining lengths must have shape [batch].")
+    commit_lengths = prefix_failure_commit_lengths(
+        failure_rate,
+        failure_budget=failure_budget,
+        valid_mask=valid_mask,
+    )
+    commit_lengths = torch.minimum(commit_lengths, remaining_lengths.clamp_min(0))
+    return first_committed_token_lengths(
+        committed_token_ids,
+        commit_lengths,
+        stop_token_id,
+    )
+
+
+def select_commit_lengths(
+    sampled_token_ids: torch.LongTensor,
+    normal_failure_rate: torch.Tensor,
+    previous_failure_rate: torch.Tensor,
+    greedy_token_ids: torch.LongTensor,
+    jump_failure_rate: torch.Tensor,
+    *,
+    ponder_steps: torch.Tensor,
+    stagnation_steps: torch.Tensor,
+    active_rows: torch.BoolTensor,
+    remaining_lengths: torch.LongTensor,
+    failure_budget: float,
+    jump_failure_budget: float,
+    stop_token_id: int | Sequence[int],
+    max_ponder_steps: int,
+    stagnation_threshold: int,
+    min_progress: float,
+    valid_mask: torch.BoolTensor | None = None,
+) -> CommitPolicyDecision:
+    """Select sampled commits or a greedy jump after stagnation.
+
+    Progress is the signed change in fused failure rate over the union of the
+    previous and current prefixes plus one blocking position.
+
+    Args:
+        sampled_token_ids: Temperature-sampled canvas tokens.
+        normal_failure_rate: Fused failure rates for the sampled tokens.
+        previous_failure_rate: Fused failure rates from the previous step.
+        greedy_token_ids: Greedy canvas tokens used for jumps.
+        jump_failure_rate: Fused failure rates for the greedy tokens.
+        ponder_steps: Per-row useful-ponder clocks.
+        stagnation_steps: Per-row stagnation clocks.
+        active_rows: Rows that are still generating.
+        remaining_lengths: Tokens still allowed on each row.
+        failure_budget: Normal commit budget.
+        jump_failure_budget: Forced-jump budget.
+        stop_token_id: Turn or EOS stop IDs.
+        max_ponder_steps: Watchdog on useful pondering.
+        stagnation_threshold: Watchdog on true stagnation.
+        min_progress: Minimum signed improvement counted as progress.
+        valid_mask: Optional canvas mask.
+
+    Returns:
+        Commit lengths, token IDs, jump flags, and updated clocks.
+    """
+
+    if not (
+        sampled_token_ids.shape
+        == normal_failure_rate.shape
+        == previous_failure_rate.shape
+        == greedy_token_ids.shape
+        == jump_failure_rate.shape
+    ):
+        raise ValueError("Sampled and greedy statistics must share [batch, canvas].")
+
+    normal = bounded_prefix_failure_commit_lengths(
+        sampled_token_ids,
+        normal_failure_rate,
+        failure_budget=failure_budget,
+        remaining_lengths=remaining_lengths,
+        stop_token_id=stop_token_id,
+        valid_mask=valid_mask,
+    )
+    canvas_length = normal_failure_rate.shape[1]
+    previous_prefix_length = prefix_failure_commit_lengths(
+        previous_failure_rate,
+        failure_budget=failure_budget,
+        valid_mask=valid_mask,
+    )
+    frontier_length = torch.maximum(previous_prefix_length, normal) + 1
+    valid_lengths = (
+        valid_mask.long().sum(dim=-1)
+        if valid_mask is not None
+        else torch.full_like(frontier_length, canvas_length)
+    )
+    frontier_length = torch.minimum(frontier_length, valid_lengths)
+    positions = torch.arange(canvas_length, device=normal_failure_rate.device)[None, :]
+    progress_mask = positions < frontier_length[:, None]
+    if valid_mask is not None:
+        progress_mask &= valid_mask
+    progress_mask &= active_rows[:, None]
+    signed_improvement = previous_failure_rate.float() - normal_failure_rate.float()
+    weights = progress_mask.float()
+    progress = (signed_improvement * weights).sum(dim=-1) / weights.sum(dim=-1).clamp_min(
+        1.0
+    )
+    next_ponder, next_stagnation = advance_trajectory_clocks(
+        ponder_steps,
+        stagnation_steps,
+        commit_lengths=normal,
+        active_rows=active_rows,
+        progress_scores=progress,
+        min_progress=min_progress,
+    )
+    jump_rows = normal.eq(0) & active_rows & should_force_trajectory_jump(
+        next_ponder,
+        next_stagnation,
+        max_ponder_steps=max_ponder_steps,
+        stagnation_threshold=stagnation_threshold,
+    )
+    jump_commit = bounded_prefix_failure_commit_lengths(
+        greedy_token_ids,
+        jump_failure_rate,
+        failure_budget=jump_failure_budget,
+        remaining_lengths=remaining_lengths,
+        stop_token_id=stop_token_id,
+        valid_mask=valid_mask,
+    )
+    committed = torch.where(jump_rows, jump_commit, normal)
+    commit_token_ids = torch.where(
+        jump_rows[:, None],
+        greedy_token_ids,
+        sampled_token_ids,
+    )
+    committed = first_committed_token_lengths(
+        commit_token_ids,
+        committed,
+        stop_token_id,
+    )
+    committed = torch.where(active_rows, committed, 0)
+    jump_rows &= committed.gt(0)
+    next_ponder = torch.where(committed.gt(0), 0, next_ponder).to(torch.int32)
+    next_stagnation = torch.where(committed.gt(0), 0, next_stagnation).to(torch.int32)
+    return CommitPolicyDecision(
+        normal_lengths=normal,
+        commit_lengths=committed,
+        commit_token_ids=commit_token_ids,
+        jump_rows=jump_rows,
+        ponder_steps=next_ponder,
+        stagnation_steps=next_stagnation,
+    )
+
+
+__all__ = [
+    "CommitPolicyDecision",
+    "bounded_prefix_failure_commit_lengths",
+    "first_committed_token_lengths",
+    "fused_commit_confidence",
+    "fused_commit_failure_rate",
+    "prefix_failure_commit_lengths",
+    "select_commit_lengths",
+]

config.json

@@ -0,0 +1,155 @@
+{
+  "architectures": [
+    "ModilifyMk1ForBlockDiffusion"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_modilify_mk1.ModilifyMk1Config",
+    "AutoModel": "modeling_modilify_mk1.ModilifyMk1Model",
+    "AutoModelForCausalLM": "modeling_modilify_mk1.ModilifyMk1ForBlockDiffusion",
+    "AutoModelForMultimodalLM": "modeling_modilify_mk1.ModilifyMk1ForBlockDiffusion"
+  },
+  "boi_token_id": 255999,
+  "bos_token_id": 2,
+  "canvas_length": 256,
+  "commit_failure_budget": 0.2,
+  "denoise_temperature": 0.8,
+  "dtype": "bfloat16",
+  "eoi_token_id": 258882,
+  "eos_token_id": [
+    1,
+    106
+  ],
+  "image_token_id": 258880,
+  "initializer_range": 0.02,
+  "jump_failure_budget": 2.0,
+  "jump_on_no_progress_after": 12,
+  "latent_dim": 1536,
+  "latent_dropout": 0.0,
+  "latent_local_attention_window": 128,
+  "latent_memory_slots": 64,
+  "latent_num_heads": 16,
+  "latent_num_layers": 4,
+  "max_ponder_steps": 64,
+  "min_trajectory_progress": 0.005,
+  "model_type": "modilify_mk1",
+  "pad_token_id": 0,
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 2,
+    "dtype": "bfloat16",
+    "eos_token_id": 1,
+    "final_logit_softcapping": 30.0,
+    "global_head_dim": 512,
+    "head_dim": 256,
+    "hidden_activation": "gelu_pytorch_tanh",
+    "hidden_size": 2816,
+    "initializer_range": 0.02,
+    "intermediate_size": 2112,
+    "layer_types": [
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "sliding_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 262144,
+    "model_type": "modilify_mk1_text",
+    "moe_intermediate_size": 704,
+    "num_attention_heads": 16,
+    "num_experts": 128,
+    "num_global_key_value_heads": 2,
+    "num_hidden_layers": 30,
+    "num_key_value_heads": 8,
+    "pad_token_id": 0,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "full_attention": {
+        "partial_rotary_factor": 0.25,
+        "rope_theta": 1000000.0,
+        "rope_type": "proportional"
+      },
+      "sliding_attention": {
+        "rope_theta": 10000.0,
+        "rope_type": "default"
+      }
+    },
+    "sliding_window": 1024,
+    "tie_word_embeddings": true,
+    "top_k_experts": 8,
+    "use_bidirectional_attention": "vision",
+    "vocab_size": 262144
+  },
+  "tie_word_embeddings": true,
+  "transformers_version": "5.14.1",
+  "turn_end_token_id": 106,
+  "vision_config": {
+    "_name_or_path": "",
+    "architectures": null,
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "chunk_size_feed_forward": 0,
+    "default_output_length": 280,
+    "dtype": "bfloat16",
+    "global_head_dim": 72,
+    "head_dim": 72,
+    "hidden_activation": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 4304,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "max_position_embeddings": 131072,
+    "model_type": "gemma4_vision",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 27,
+    "num_key_value_heads": 16,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "patch_size": 16,
+    "pooling_kernel_size": 3,
+    "position_embedding_size": 10240,
+    "problem_type": null,
+    "return_dict": true,
+    "rms_norm_eps": 1e-06,
+    "rope_parameters": {
+      "rope_theta": 100.0,
+      "rope_type": "default"
+    },
+    "standardize": true,
+    "use_clipped_linears": false
+  },
+  "repetition_penalty": 1.0
+}

configuration_modilify_mk1.py

@@ -0,0 +1,169 @@
+# Copyright 2026 Modilify
+# SPDX-License-Identifier: LicenseRef-Modilify-Open-Model-1.0
+"""Configuration classes for Modilify Mk1."""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+from transformers.models.diffusion_gemma import (
+    DiffusionGemmaConfig,
+    DiffusionGemmaTextConfig,
+)
+
+
+class ModilifyMk1TextConfig(DiffusionGemmaTextConfig):
+    """Text configuration for the Modilify Mk1 decoder.
+
+    This class preserves the standard DiffusionGemma text schema while giving
+    the exported model an independent, stable model type.
+    """
+
+    model_type = "modilify_mk1_text"
+
+
+class ModilifyMk1Config(DiffusionGemmaConfig):
+    """Serializable multimodal inference configuration for Modilify Mk1.
+
+    Args:
+        text_config: DiffusionGemma text configuration or its serialized form.
+        vision_config: Gemma 4 vision configuration or its serialized form.
+        denoise_temperature: Sampling temperature used at every denoising step.
+        commit_failure_budget: Maximum cumulative failure risk for normal commits.
+        jump_failure_budget: Maximum cumulative failure risk for forced jumps.
+        latent_dim: Width of the recurrent latent state.
+        latent_memory_slots: Number of persistent latent memory slots.
+        latent_num_layers: Number of latent Transformer blocks.
+        latent_num_heads: Number of latent attention heads.
+        latent_local_attention_window: Local token-attention radius.
+        latent_dropout: Latent Transformer dropout probability.
+        jump_on_no_progress_after: Stagnation steps before a forced jump.
+        max_ponder_steps: Maximum denoising iterations per requested token.
+        min_trajectory_progress: Minimum fused-risk improvement counted as progress.
+        turn_end_token_id: Native Gemma turn terminator.
+        repetition_penalty: Transformers-style repetition penalty. ``1.0`` disables
+            it.
+        kwargs: Standard DiffusionGemma configuration values.
+    """
+
+    model_type = "modilify_mk1"
+    sub_configs = {
+        "text_config": ModilifyMk1TextConfig,
+        **{
+            key: value
+            for key, value in DiffusionGemmaConfig.sub_configs.items()
+            if key != "text_config"
+        },
+    }
+
+    def __init__(
+        self,
+        text_config: (
+            ModilifyMk1TextConfig
+            | DiffusionGemmaTextConfig
+            | dict[str, Any]
+            | None
+        ) = None,
+        vision_config: Any | dict[str, Any] | None = None,
+        *,
+        denoise_temperature: float = 0.8,
+        commit_failure_budget: float = 0.2,
+        jump_failure_budget: float = 2.0,
+        latent_dim: int = 1536,
+        latent_memory_slots: int = 64,
+        latent_num_layers: int = 4,
+        latent_num_heads: int = 16,
+        latent_local_attention_window: int = 128,
+        latent_dropout: float = 0.0,
+        jump_on_no_progress_after: int = 12,
+        max_ponder_steps: int = 64,
+        min_trajectory_progress: float = 0.005,
+        turn_end_token_id: int = 106,
+        repetition_penalty: float = 1.0,
+        **kwargs: Any,
+    ) -> None:
+        kwargs.pop("model_type", None)
+        kwargs.pop("fused_entropy_weight", None)
+        kwargs.pop("vocab_chunk_size", None)
+        if isinstance(text_config, DiffusionGemmaTextConfig):
+            text_payload = text_config.to_dict()
+            text_payload.pop("model_type", None)
+            text_config = ModilifyMk1TextConfig(**text_payload)
+        elif isinstance(text_config, dict):
+            text_payload = dict(text_config)
+            text_payload.pop("model_type", None)
+            text_config = ModilifyMk1TextConfig(**text_payload)
+        elif text_config is None:
+            text_config = ModilifyMk1TextConfig()
+
+        self.denoise_temperature = float(denoise_temperature)
+        self.commit_failure_budget = float(commit_failure_budget)
+        self.jump_failure_budget = float(jump_failure_budget)
+        self.latent_dim = int(latent_dim)
+        self.latent_memory_slots = int(latent_memory_slots)
+        self.latent_num_layers = int(latent_num_layers)
+        self.latent_num_heads = int(latent_num_heads)
+        self.latent_local_attention_window = int(latent_local_attention_window)
+        self.latent_dropout = float(latent_dropout)
+        self.jump_on_no_progress_after = int(jump_on_no_progress_after)
+        self.max_ponder_steps = int(max_ponder_steps)
+        self.min_trajectory_progress = float(min_trajectory_progress)
+        self.turn_end_token_id = int(turn_end_token_id)
+        self.repetition_penalty = float(repetition_penalty)
+        super().__init__(
+            text_config=text_config,
+            vision_config=vision_config,
+            **kwargs,
+        )
+        self.model_type = type(self).model_type
+        if not hasattr(self, "eos_token_id"):
+            self.eos_token_id = self.text_config.eos_token_id
+        if not hasattr(self, "pad_token_id"):
+            self.pad_token_id = self.text_config.pad_token_id
+        if not hasattr(self, "bos_token_id"):
+            self.bos_token_id = self.text_config.bos_token_id
+        self._validate_modilify()
+
+    def _validate_modilify(self) -> None:
+        """Validate inference extension values."""
+
+        policy_values = (
+            self.denoise_temperature,
+            self.commit_failure_budget,
+            self.jump_failure_budget,
+            self.min_trajectory_progress,
+            self.repetition_penalty,
+        )
+        if any(not math.isfinite(value) for value in policy_values):
+            raise ValueError("Modilify Mk1 policy values must be finite.")
+        positive = (
+            self.denoise_temperature,
+            self.commit_failure_budget,
+            self.jump_failure_budget,
+            self.latent_dim,
+            self.latent_memory_slots,
+            self.latent_num_layers,
+            self.latent_num_heads,
+            self.latent_local_attention_window,
+            self.jump_on_no_progress_after,
+            self.max_ponder_steps,
+            self.repetition_penalty,
+        )
+        if any(value <= 0 for value in positive):
+            raise ValueError(
+                "Modilify Mk1 dimensions, budgets, intervals, and "
+                "`repetition_penalty` must be positive."
+            )
+        if self.latent_dim % self.latent_num_heads:
+            raise ValueError("`latent_dim` must be divisible by `latent_num_heads`.")
+        if not 0.0 <= self.latent_dropout < 1.0:
+            raise ValueError("`latent_dropout` must be in [0, 1).")
+        if self.min_trajectory_progress < 0:
+            raise ValueError("`min_trajectory_progress` must be non-negative.")
+
+
+ModilifyMk1Config.register_for_auto_class()
+
+
+__all__ = ["ModilifyMk1Config", "ModilifyMk1TextConfig"]

generation_config.json

@@ -0,0 +1,16 @@
+{
+  "denoise_temperature": 0.8,
+  "eos_token_id": [
+    1,
+    106
+  ],
+  "max_denoising_steps": null,
+  "max_new_tokens": 256,
+  "one_token_per_denoise_step": false,
+  "repetition_penalty": 1.0,
+  "return_dict_in_generate": true,
+  "t_max": 0.8,
+  "t_min": 0.8,
+  "transformers_version": "5.14.1",
+  "turn_end_token_id": 106
+}

generation_modilify_mk1.py

@@ -0,0 +1,835 @@
+# Copyright 2026 Modilify
+# SPDX-License-Identifier: LicenseRef-Modilify-Open-Model-1.0
+"""Rolling generation for latent-memory Modilify Mk1."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass, replace
+import math
+from typing import Any
+
+import torch
+from transformers.cache_utils import Cache
+from transformers.generation import LogitsProcessorList
+from transformers.generation.streamers import BaseStreamer
+from transformers.modeling_outputs import ModelOutput
+
+from transformers.models.diffusion_gemma import (
+    DiffusionGemmaGenerationConfig,
+    DiffusionGemmaGenerationMixin,
+)
+from .commit_policy import fused_commit_failure_rate, select_commit_lengths
+from .latent_deliberation import LatentDeliberationState
+
+
+def _flatten_token_ids(*values: object) -> set[int]:
+    """Normalize scalar and sequence token-ID configuration values."""
+
+    token_ids: set[int] = set()
+    for value in values:
+        if value is None:
+            continue
+        if isinstance(value, int):
+            token_ids.add(int(value))
+            continue
+        if isinstance(value, (list, tuple, set)):
+            token_ids.update(int(token_id) for token_id in value if token_id is not None)
+    return token_ids
+
+
+def _add_repetition_history(
+    history: torch.BoolTensor,
+    token_ids: torch.LongTensor,
+    eligible: torch.BoolTensor,
+    excluded_token_ids: set[int],
+) -> None:
+    """Add eligible row-local token IDs to a compact ``[batch, vocab]`` history."""
+
+    if token_ids.shape != eligible.shape or token_ids.shape[0] != history.shape[0]:
+        raise ValueError("Repetition history token and eligibility shapes must match.")
+    eligible = eligible.clone()
+    for token_id in excluded_token_ids:
+        eligible &= token_ids.ne(token_id)
+    if not bool(eligible.any()):
+        return
+    rows = torch.arange(history.shape[0], device=history.device)[:, None]
+    rows = rows.expand_as(token_ids)
+    history[rows[eligible], token_ids[eligible]] = True
+
+
+class ModilifyMk1GenerationConfig(DiffusionGemmaGenerationConfig):
+    """Generation controls for the Modilify Mk1 commit policy.
+
+    Args:
+        turn_end_token_id: Token that closes a native Gemma turn.
+        denoise_temperature: Sampling temperature used for every canvas step.
+        repetition_penalty: Transformers-style repetition penalty. ``1.0``
+            disables it.
+        kwargs: Standard DiffusionGemma generation arguments.
+    """
+
+    def __init__(
+        self,
+        *,
+        turn_end_token_id: int | None = None,
+        denoise_temperature: float = 0.8,
+        repetition_penalty: float = 1.0,
+        **kwargs: Any,
+    ) -> None:
+        self.turn_end_token_id = turn_end_token_id
+        self.denoise_temperature = float(denoise_temperature)
+        self.repetition_penalty = float(repetition_penalty)
+        kwargs.pop("one_token_per_denoise_step", None)
+        kwargs["t_min"] = self.denoise_temperature
+        kwargs["t_max"] = self.denoise_temperature
+        super().__init__(**kwargs)
+        self.one_token_per_denoise_step = False
+
+    def update(self, **kwargs: Any) -> dict[str, Any]:
+        """Apply standard generation overrides and supported local overrides."""
+
+        if "denoise_temperature" in kwargs:
+            self.denoise_temperature = float(kwargs.pop("denoise_temperature"))
+        if "repetition_penalty" in kwargs:
+            self.repetition_penalty = float(kwargs.pop("repetition_penalty"))
+        kwargs["t_min"] = self.denoise_temperature
+        kwargs["t_max"] = self.denoise_temperature
+        unused = super().update(**kwargs)
+        self.one_token_per_denoise_step = False
+        self.t_min = self.denoise_temperature
+        self.t_max = self.denoise_temperature
+        return unused
+
+    def validate(self, **kwargs: Any) -> None:
+        """Validate generation values, including a single temperature.
+
+        DiffusionGemma's parent validator requires a non-empty temperature
+        interval. Modilify Mk1 uses one temperature, so ``t_min == t_max`` is
+        validated here.
+        """
+
+        del kwargs
+        if (
+            not math.isfinite(self.denoise_temperature)
+            or self.denoise_temperature <= 0
+        ):
+            raise ValueError("`denoise_temperature` must be positive.")
+        if not math.isfinite(self.repetition_penalty) or self.repetition_penalty <= 0:
+            raise ValueError("`repetition_penalty` must be a finite positive number.")
+        if self.max_denoising_steps is not None and (
+            not isinstance(self.max_denoising_steps, int)
+            or self.max_denoising_steps <= 0
+        ):
+            raise ValueError("`max_denoising_steps` must be a positive integer.")
+        if self.turn_end_token_id is not None and (
+            not isinstance(self.turn_end_token_id, int) or self.turn_end_token_id < 0
+        ):
+            raise ValueError("`turn_end_token_id` must be a non-negative integer.")
+
+    @classmethod
+    def from_model_config(cls, model_config: Any) -> "ModilifyMk1GenerationConfig":
+        """Build generation defaults from a model configuration."""
+
+        return cls(
+            turn_end_token_id=model_config.turn_end_token_id,
+            denoise_temperature=model_config.denoise_temperature,
+            repetition_penalty=getattr(model_config, "repetition_penalty", 1.0),
+            eos_token_id=getattr(
+                model_config,
+                "eos_token_id",
+                model_config.text_config.eos_token_id,
+            ),
+        )
+
+    @staticmethod
+    def _get_default_generation_params() -> dict[str, object]:
+        """Return defaults with no inherited entropy or readiness controls."""
+
+        return {
+            "max_new_tokens": 256,
+            "max_denoising_steps": 48,
+            "t_min": 0.8,
+            "t_max": 0.8,
+        }
+
+
+@dataclass
+class ModilifyMk1GenerationOutput(ModelOutput):
+    """Structured result returned by rolling block-diffusion generation."""
+
+    sequences: torch.LongTensor
+    generated_lengths: torch.LongTensor | None = None
+    tokens_per_forward: torch.FloatTensor | None = None
+    past_key_values: Cache | None = None
+    stop_reason: str | tuple[str, ...] | None = None
+    committed_tokens: int | torch.LongTensor | None = None
+    denoise_steps: int | torch.LongTensor | None = None
+    no_progress_steps: int | torch.LongTensor | None = None
+    jump_count: int | torch.LongTensor | None = None
+    forced_jump_bad_count: int | torch.LongTensor | None = None
+    heavy_forward_count: int | torch.LongTensor | None = None
+    latent_context_update_count: int | torch.LongTensor | None = None
+    average_commit_len: float | torch.FloatTensor | None = None
+    state_shift_count: int | torch.LongTensor | None = None
+    latent_memory_norm: float | torch.FloatTensor | None = None
+    state_retention_score: float | torch.FloatTensor | None = None
+    logits: None = None
+    scores: None = None
+    hidden_states: None = None
+
+
+@dataclass
+class _RollingState:
+    """All real iterative state; no vocabulary-sized tensor is retained."""
+
+    canvas: torch.LongTensor
+    confidence: torch.FloatTensor
+    entropy: torch.FloatTensor
+    age: torch.IntTensor
+    latent_state: LatentDeliberationState
+    history_hidden_state: torch.FloatTensor | None
+
+
+def _retain_denoise_proposals(proposal: torch.LongTensor) -> torch.LongTensor:
+    """Keep every latest denoise token; confidence controls commit, not writeback."""
+
+    if proposal.ndim != 2:
+        raise ValueError("Denoise proposals must have shape [batch, canvas].")
+    return proposal.clone()
+
+
+class _NoiseCanvasSampler:
+    """Uniform diffusion noise source with no commit-policy responsibilities."""
+
+    def __init__(self, *, canvas_length: int, vocab_size: int) -> None:
+        self.canvas_length = int(canvas_length)
+        self.vocab_size = int(vocab_size)
+        self.initial_entropy = math.log(self.vocab_size)
+
+    def initialize_canvas(
+        self,
+        batch_size: int,
+        device: torch.device,
+    ) -> torch.LongTensor:
+        """Sample a uniformly random starting canvas.
+
+        Args:
+            batch_size: Number of canvases to create.
+            device: Device on which token IDs are allocated.
+
+        Returns:
+            Random token IDs with shape ``[batch_size, canvas_length]``.
+        """
+
+        return torch.randint(
+            self.vocab_size,
+            (batch_size, self.canvas_length),
+            device=device,
+        )
+
+
+class ModilifyMk1GenerationMixin(DiffusionGemmaGenerationMixin):
+    """Transformers-compatible rolling latent-deliberation generator."""
+
+    def _prepare_sampler(
+        self,
+        generation_config: ModilifyMk1GenerationConfig,
+        canvas_length: int | None = None,
+    ) -> _NoiseCanvasSampler:
+        del generation_config
+        return _NoiseCanvasSampler(
+            canvas_length=canvas_length or self.config.canvas_length,
+            vocab_size=self.config.text_config.vocab_size,
+        )
+
+    @staticmethod
+    def _shift_state_rows(
+        state: _RollingState,
+        commit_lengths: torch.LongTensor,
+        sampler: _NoiseCanvasSampler,
+    ) -> _RollingState:
+        """Shift every rolling row by its own committed prefix length."""
+
+        batch_size, canvas_length = state.canvas.shape
+        if commit_lengths.shape != (batch_size,):
+            raise ValueError("Commit lengths must have shape [batch].")
+        if not bool(commit_lengths.gt(0).any()):
+            return state
+        positions = torch.arange(canvas_length, device=state.canvas.device)[None, :]
+        source = positions + commit_lengths[:, None]
+        retained = source.lt(canvas_length)
+
+        def shift(value: torch.Tensor, fill_value: float | int = 0) -> torch.Tensor:
+            index = source.clamp_max(canvas_length - 1)
+            index = index.view(
+                batch_size, canvas_length, *([1] * (value.ndim - 2))
+            ).expand_as(value)
+            gathered = value.gather(1, index)
+            mask = retained.view(
+                batch_size, canvas_length, *([1] * (value.ndim - 2))
+            )
+            fill = torch.as_tensor(fill_value, device=value.device, dtype=value.dtype)
+            return torch.where(mask, gathered, fill)
+
+        tail = sampler.initialize_canvas(batch_size, state.canvas.device)
+        canvas = torch.cat((state.canvas, tail), dim=1).gather(1, source)
+        unknown_entropy = float(sampler.initial_entropy)
+        latent = state.latent_state
+        committed = commit_lengths.gt(0)
+        shifted_latent = LatentDeliberationState(
+            token_latents=shift(latent.token_latents),
+            memory_slots=latent.memory_slots.clone(),
+            confidence=shift(latent.confidence),
+            entropy=shift(latent.entropy, unknown_entropy),
+            age=shift(latent.age),
+            token_changed=shift(latent.token_changed),
+            confidence_delta=shift(latent.confidence_delta),
+            entropy_delta=shift(latent.entropy_delta),
+            ponder_steps=torch.where(
+                committed, torch.zeros_like(latent.ponder_steps), latent.ponder_steps
+            ),
+            stagnation_steps=torch.where(
+                committed,
+                torch.zeros_like(latent.stagnation_steps),
+                latent.stagnation_steps,
+            ),
+        )
+        return _RollingState(
+            canvas=canvas,
+            confidence=shift(state.confidence),
+            entropy=shift(state.entropy, unknown_entropy),
+            age=shift(state.age),
+            latent_state=shifted_latent,
+            history_hidden_state=(
+                None
+                if state.history_hidden_state is None
+                else shift(state.history_hidden_state)
+            ),
+        )
+
+    @staticmethod
+    def _merge_state_rows(
+        previous: _RollingState,
+        updated: _RollingState,
+        update_mask: torch.BoolTensor,
+    ) -> _RollingState:
+        """Advance active rows while leaving completed rows unchanged."""
+
+        def choose(old: torch.Tensor, new: torch.Tensor) -> torch.Tensor:
+            mask = update_mask.view(
+                update_mask.shape[0],
+                *([1] * (old.ndim - 1)),
+            )
+            return torch.where(mask, new, old)
+
+        old_latent = previous.latent_state
+        new_latent = updated.latent_state
+        latent = LatentDeliberationState(
+            token_latents=choose(old_latent.token_latents, new_latent.token_latents),
+            memory_slots=choose(old_latent.memory_slots, new_latent.memory_slots),
+            confidence=choose(old_latent.confidence, new_latent.confidence),
+            entropy=choose(old_latent.entropy, new_latent.entropy),
+            age=choose(old_latent.age, new_latent.age),
+            token_changed=choose(old_latent.token_changed, new_latent.token_changed),
+            confidence_delta=choose(
+                old_latent.confidence_delta, new_latent.confidence_delta
+            ),
+            entropy_delta=choose(old_latent.entropy_delta, new_latent.entropy_delta),
+            ponder_steps=choose(old_latent.ponder_steps, new_latent.ponder_steps),
+            stagnation_steps=choose(
+                old_latent.stagnation_steps, new_latent.stagnation_steps
+            ),
+        )
+        history = previous.history_hidden_state
+        if updated.history_hidden_state is not None:
+            history = (
+                updated.history_hidden_state
+                if history is None
+                else choose(history, updated.history_hidden_state)
+            )
+        return _RollingState(
+            canvas=choose(previous.canvas, updated.canvas),
+            confidence=choose(previous.confidence, updated.confidence),
+            entropy=choose(previous.entropy, updated.entropy),
+            age=choose(previous.age, updated.age),
+            latent_state=latent,
+            history_hidden_state=history,
+        )
+
+    @torch.inference_mode()
+    def generate(
+        self,
+        input_ids: torch.LongTensor | None = None,
+        past_key_values: Cache | None = None,
+        streamer: BaseStreamer | None = None,
+        generation_config: ModilifyMk1GenerationConfig | None = None,
+        logits_processor: LogitsProcessorList | None = None,
+        **kwargs,
+    ) -> ModilifyMk1GenerationOutput:
+        """Generate one or more responses with rolling block diffusion.
+
+        Args:
+            input_ids: Tokenized prompts with shape ``[batch, sequence]``.
+            past_key_values: Optional existing encoder cache.
+            streamer: Optional standard Transformers token streamer.
+            generation_config: Generation limits and token IDs.
+            logits_processor: Unsupported custom logits processors.
+            **kwargs: Standard multimodal encoder inputs and generation values.
+
+        Returns:
+            Generated sequences and diffusion diagnostics.
+
+        Raises:
+            ValueError: If inputs are invalid or unsupported logits processing
+                is requested.
+        """
+
+        generation_config, model_kwargs = self._prepare_generation_config(
+            generation_config,
+            **kwargs,
+        )
+        if input_ids is None or input_ids.ndim != 2 or input_ids.shape[0] < 1:
+            raise ValueError(
+                "Modilify Mk1 generation requires `input_ids` with shape "
+                "[batch, sequence]."
+            )
+        if logits_processor:
+            raise ValueError(
+                "Modilify Mk1 samples the configured temperature distribution "
+                "and does not accept custom logits processors."
+            )
+        batch_size, input_width = input_ids.shape
+        if batch_size > 1 and streamer is not None:
+            raise ValueError("Streamers currently support batch size 1 only.")
+        if batch_size > 1 and past_key_values is not None:
+            raise ValueError("Batched generation requires a fresh KV cache.")
+        device = input_ids.device
+        dtype = self.model.decoder.embed_tokens.weight.dtype
+        canvas_length = self.config.canvas_length
+        cached_length = (
+            past_key_values.get_seq_length() if past_key_values is not None else 0
+        )
+        repetition_penalty = float(generation_config.repetition_penalty)
+        repetition_enabled = repetition_penalty != 1.0
+        if repetition_enabled and cached_length:
+            raise ValueError(
+                "Repetition penalty requires a fresh KV cache so the complete "
+                "prompt token history is available."
+            )
+        _, max_new_tokens = self._prepare_generated_length(
+            generation_config, cached_length + input_width
+        )
+        max_iterations = max(1, max_new_tokens * int(self.config.max_ponder_steps))
+        if past_key_values is None:
+            past_key_values = self._prepare_cache_for_generation(
+                generation_config,
+                batch_size=batch_size,
+                max_length=input_width + batch_size * max_new_tokens,
+            )
+        expected_mask_width = cached_length + input_width
+        cache_attention_mask = model_kwargs.pop(
+            "attention_mask",
+            torch.ones(
+                batch_size, expected_mask_width, dtype=torch.bool, device=device
+            ),
+        ).bool()
+        if cache_attention_mask.shape != (batch_size, expected_mask_width):
+            raise ValueError(
+                "`attention_mask` must have shape [batch, cached_length + sequence]."
+            )
+        provided_position_ids = model_kwargs.pop("position_ids", None)
+        if provided_position_ids is not None:
+            if provided_position_ids.shape != input_ids.shape:
+                raise ValueError("`position_ids` must have the same shape as `input_ids`.")
+            prompt_positions = provided_position_ids.to(device=device, dtype=torch.int32)
+        elif cached_length:
+            prompt_positions = torch.arange(
+                cached_length,
+                cached_length + input_width,
+                device=device,
+                dtype=torch.int32,
+            ).unsqueeze(0)
+        else:
+            input_mask = cache_attention_mask[:, -input_width:]
+            prompt_positions = (
+                input_mask.long().cumsum(dim=-1).sub(1).clamp_min(0).to(torch.int32)
+            )
+        logical_lengths = cache_attention_mask.long().sum(dim=-1)
+        if input_width:
+            encoder_keys = ("pixel_values", "mm_token_type_ids", "image_position_ids")
+            encoder_kwargs = {
+                key: model_kwargs.pop(key)
+                for key in encoder_keys
+                if key in model_kwargs
+            }
+            past_key_values = self.model.encoder(
+                input_ids=input_ids,
+                attention_mask=cache_attention_mask,
+                past_key_values=past_key_values,
+                position_ids=prompt_positions,
+                **encoder_kwargs,
+            ).past_key_values
+
+        sampler = self._prepare_sampler(generation_config, canvas_length)
+        latent = LatentDeliberationState.empty(
+            batch_size=batch_size,
+            canvas_length=canvas_length,
+            latent_dim=self.config.latent_dim,
+            memory_slots=self.config.latent_memory_slots,
+            device=device,
+            dtype=dtype,
+        )
+        state = _RollingState(
+            canvas=sampler.initialize_canvas(batch_size, device),
+            confidence=torch.zeros(
+                batch_size, canvas_length, device=device, dtype=torch.float32
+            ),
+            entropy=torch.full(
+                (batch_size, canvas_length),
+                math.log(self.config.text_config.vocab_size),
+                device=device,
+                dtype=torch.float32,
+            ),
+            age=torch.zeros(
+                batch_size, canvas_length, device=device, dtype=torch.int32
+            ),
+            latent_state=latent,
+            history_hidden_state=None,
+        )
+        turn_end = (
+            self.config.turn_end_token_id
+            if generation_config.turn_end_token_id is None
+            else generation_config.turn_end_token_id
+        )
+        configured_eos = generation_config.eos_token_id
+        if configured_eos is None:
+            configured_eos = self.config.eos_token_id
+        if isinstance(configured_eos, int):
+            configured_eos = [configured_eos]
+        stop_token_ids = tuple(
+            dict.fromkeys((int(turn_end), *(int(value) for value in configured_eos or ())))
+        )
+        pad_token_id = generation_config.pad_token_id
+        if pad_token_id is None:
+            pad_token_id = getattr(self.config, "pad_token_id", None)
+        if isinstance(pad_token_id, (list, tuple)):
+            pad_token_id = pad_token_id[0]
+        pad_token_id = int(0 if pad_token_id is None else pad_token_id)
+        excluded_repetition_token_ids = _flatten_token_ids(
+            generation_config.pad_token_id,
+            generation_config.bos_token_id,
+            generation_config.eos_token_id,
+            generation_config.turn_end_token_id,
+            getattr(self.config, "image_token_id", None),
+        )
+        repetition_history = None
+        if repetition_enabled:
+            repetition_history = torch.zeros(
+                (batch_size, self.config.text_config.vocab_size),
+                dtype=torch.bool,
+                device=device,
+            )
+            _add_repetition_history(
+                repetition_history,
+                input_ids,
+                cache_attention_mask[:, -input_width:],
+                excluded_repetition_token_ids,
+            )
+        generated = torch.full(
+            (batch_size, max_new_tokens),
+            pad_token_id,
+            dtype=input_ids.dtype,
+            device=device,
+        )
+        committed = torch.zeros(batch_size, dtype=torch.long, device=device)
+        denoise_steps = torch.zeros_like(committed)
+        jumps = torch.zeros_like(committed)
+        forced_jump_tokens = torch.zeros_like(committed)
+        shifts = torch.zeros_like(committed)
+        retention_scores = torch.zeros(batch_size, dtype=torch.float32, device=device)
+        stop_codes = torch.zeros_like(committed)
+        active_rows = torch.ones(batch_size, dtype=torch.bool, device=device)
+        canvas_positions = torch.arange(canvas_length, device=device)[None, :]
+        if streamer is not None:
+            streamer.put(input_ids.cpu())
+
+        while bool(active_rows.any()):
+            decoder_positions = (logical_lengths[:, None] + canvas_positions).to(
+                torch.int32
+            )
+            denoise_steps += active_rows.long()
+            decoder_attention_mask = torch.cat(
+                (
+                    cache_attention_mask,
+                    torch.ones(
+                        batch_size,
+                        canvas_length,
+                        dtype=torch.bool,
+                        device=device,
+                    ),
+                ),
+                dim=-1,
+            )
+            output = self(
+                input_ids=None,
+                past_key_values=past_key_values,
+                decoder_input_ids=state.canvas,
+                previous_confidence=state.confidence,
+                previous_entropy=state.entropy,
+                token_age=state.age,
+                latent_state=state.latent_state,
+                history_hidden_state=state.history_hidden_state,
+                decoder_position_ids=decoder_positions,
+                decoder_read_cache=True,
+                decoder_attention_mask=decoder_attention_mask,
+                return_proposal_statistics=True,
+                denoise_temperature=generation_config.denoise_temperature,
+                repetition_token_mask=repetition_history,
+                repetition_penalty=repetition_penalty,
+                **model_kwargs,
+            )
+            if any(
+                value is None
+                for value in (
+                    output.proposal,
+                    output.proposal_confidence,
+                    output.token_entropy,
+                    output.greedy_proposal,
+                    output.greedy_confidence,
+                )
+            ):
+                raise RuntimeError("Model forward did not return proposal statistics.")
+            proposal = output.proposal
+            proposal_confidence = output.proposal_confidence
+            token_entropy = output.token_entropy
+            greedy_proposal = output.greedy_proposal
+            greedy_confidence = output.greedy_confidence
+            next_canvas = _retain_denoise_proposals(proposal)
+            next_confidence = proposal_confidence.float()
+            next_latent = replace(
+                output.next_latent_state,
+                confidence=next_confidence.detach().float(),
+                entropy=token_entropy.detach().float(),
+                age=state.age + 1,
+                token_changed=next_canvas.ne(state.canvas).detach().float(),
+                confidence_delta=next_confidence.detach().float() - state.confidence,
+                entropy_delta=token_entropy.detach().float() - state.entropy,
+            )
+            next_state = _RollingState(
+                canvas=next_canvas,
+                confidence=next_confidence,
+                entropy=token_entropy,
+                age=state.age + 1,
+                latent_state=next_latent,
+                history_hidden_state=output.heavy_hidden_state,
+            )
+            next_state = self._merge_state_rows(state, next_state, active_rows)
+            remaining = torch.tensor(
+                max_new_tokens, device=device, dtype=torch.long
+            ).sub(committed)
+            vocab_size = self.config.text_config.vocab_size
+            normal_failure_rate = fused_commit_failure_rate(
+                proposal_confidence, token_entropy, vocab_size=vocab_size
+            )
+            jump_failure_rate = fused_commit_failure_rate(
+                greedy_confidence, token_entropy, vocab_size=vocab_size
+            )
+            previous_failure_rate = fused_commit_failure_rate(
+                state.confidence, state.entropy, vocab_size=vocab_size
+            )
+            policy_decision = select_commit_lengths(
+                sampled_token_ids=proposal,
+                normal_failure_rate=normal_failure_rate,
+                previous_failure_rate=previous_failure_rate,
+                greedy_token_ids=greedy_proposal,
+                jump_failure_rate=jump_failure_rate,
+                ponder_steps=state.latent_state.ponder_steps,
+                stagnation_steps=state.latent_state.stagnation_steps,
+                active_rows=active_rows,
+                remaining_lengths=remaining,
+                failure_budget=self.config.commit_failure_budget,
+                jump_failure_budget=self.config.jump_failure_budget,
+                stop_token_id=stop_token_ids,
+                max_ponder_steps=self.config.max_ponder_steps,
+                stagnation_threshold=self.config.jump_on_no_progress_after,
+                min_progress=self.config.min_trajectory_progress,
+            )
+            next_ponder = policy_decision.ponder_steps
+            next_stagnation = policy_decision.stagnation_steps
+            commit_lengths = policy_decision.commit_lengths
+            jump_rows = policy_decision.jump_rows
+            jumps += jump_rows.long()
+            forced_jump_tokens += torch.where(
+                jump_rows, commit_lengths, torch.zeros_like(commit_lengths)
+            )
+            commit_positions = canvas_positions.lt(commit_lengths[:, None])
+            if bool(jump_rows.any()):
+                next_state = replace(
+                    next_state,
+                    canvas=torch.where(
+                        commit_positions & jump_rows[:, None],
+                        policy_decision.commit_token_ids,
+                        next_state.canvas,
+                    ),
+                )
+            next_state = replace(
+                next_state,
+                latent_state=replace(
+                    next_state.latent_state,
+                    ponder_steps=next_ponder,
+                    stagnation_steps=next_stagnation,
+                ),
+            )
+            commit_token_ids = policy_decision.commit_token_ids
+            before = committed.clone()
+            write_rows = torch.arange(batch_size, device=device)[:, None].expand_as(
+                commit_token_ids
+            )
+            write_positions = before[:, None] + canvas_positions
+            generated[
+                write_rows[commit_positions], write_positions[commit_positions]
+            ] = commit_token_ids[commit_positions]
+            if repetition_history is not None:
+                _add_repetition_history(
+                    repetition_history,
+                    commit_token_ids,
+                    commit_positions,
+                    excluded_repetition_token_ids,
+                )
+
+            commit_width = int(commit_lengths.max())
+            if commit_width:
+                block_mask = torch.arange(commit_width, device=device)[None, :].lt(
+                    commit_lengths[:, None]
+                )
+                committed_block = torch.where(
+                    block_mask,
+                    commit_token_ids[:, :commit_width],
+                    torch.full(
+                        (batch_size, commit_width),
+                        pad_token_id,
+                        device=device,
+                        dtype=input_ids.dtype,
+                    ),
+                )
+                block_positions = (
+                    logical_lengths[:, None] + canvas_positions[:, :commit_width]
+                ).to(torch.int32)
+                block_positions = torch.where(
+                    block_mask, block_positions, torch.zeros_like(block_positions)
+                )
+                cache_attention_mask = torch.cat(
+                    (cache_attention_mask, block_mask), dim=-1
+                )
+                past_key_values = self.model.encoder(
+                    input_ids=committed_block,
+                    attention_mask=cache_attention_mask,
+                    past_key_values=past_key_values,
+                    position_ids=block_positions,
+                ).past_key_values
+                if streamer is not None:
+                    streamer.put(committed_block.cpu())
+            committed += commit_lengths
+            logical_lengths += commit_lengths
+            committed_rows = commit_lengths.gt(0)
+            shifts += committed_rows.long()
+            state = self._shift_state_rows(next_state, commit_lengths, sampler)
+            retention_scores += committed_rows.float()
+
+            turn_hits = (commit_token_ids.eq(turn_end) & commit_positions).any(dim=-1)
+            eos_hits = torch.zeros_like(turn_hits)
+            for token_id in stop_token_ids:
+                if token_id != turn_end:
+                    eos_hits |= (commit_token_ids.eq(token_id) & commit_positions).any(
+                        dim=-1
+                    )
+            stop_codes = torch.where(
+                stop_codes.eq(0) & turn_hits,
+                torch.ones_like(stop_codes),
+                stop_codes,
+            )
+            stop_codes = torch.where(
+                stop_codes.eq(0) & eos_hits,
+                torch.full_like(stop_codes, 2),
+                stop_codes,
+            )
+            stop_codes = torch.where(
+                stop_codes.eq(0) & committed.ge(max_new_tokens),
+                torch.full_like(stop_codes, 3),
+                stop_codes,
+            )
+            if generation_config.max_denoising_steps is not None:
+                stop_codes = torch.where(
+                    stop_codes.eq(0)
+                    & denoise_steps.ge(generation_config.max_denoising_steps),
+                    torch.full_like(stop_codes, 4),
+                    stop_codes,
+                )
+            stop_codes = torch.where(
+                stop_codes.eq(0) & denoise_steps.ge(max_iterations),
+                torch.full_like(stop_codes, 5),
+                stop_codes,
+            )
+            active_rows = stop_codes.eq(0)
+
+        output_width = int(committed.max())
+        sequences = torch.cat((input_ids, generated[:, :output_width]), dim=-1)
+        if streamer is not None:
+            streamer.end()
+        reason_names = {
+            1: "turn_end",
+            2: "eos",
+            3: "max_new_tokens",
+            4: "max_denoising_steps",
+            5: "episode_watchdog",
+        }
+        stop_reasons = tuple(
+            reason_names.get(code, "unknown")
+            for code in stop_codes.detach().cpu().tolist()
+        )
+        tokens_per_forward = committed.float() / denoise_steps.clamp_min(1).float()
+        average_commit_len = committed.float() / shifts.clamp_min(1).float()
+        latent_memory_norm = (
+            state.latent_state.memory_slots.float().norm(dim=-1).mean(dim=-1)
+        )
+        state_retention_score = retention_scores / shifts.clamp_min(1).float()
+
+        def scalar_or_tensor(
+            value: torch.Tensor,
+            *,
+            floating: bool = False,
+        ) -> int | float | torch.Tensor:
+            if batch_size > 1:
+                return value
+            item = value[0].item()
+            return float(item) if floating else int(item)
+
+        return ModilifyMk1GenerationOutput(
+            sequences=sequences,
+            generated_lengths=committed.clone(),
+            tokens_per_forward=tokens_per_forward,
+            past_key_values=past_key_values,
+            stop_reason=stop_reasons[0] if batch_size == 1 else stop_reasons,
+            committed_tokens=scalar_or_tensor(committed),
+            denoise_steps=scalar_or_tensor(denoise_steps),
+            no_progress_steps=scalar_or_tensor(state.latent_state.stagnation_steps),
+            jump_count=scalar_or_tensor(jumps),
+            forced_jump_bad_count=scalar_or_tensor(forced_jump_tokens),
+            heavy_forward_count=scalar_or_tensor(denoise_steps),
+            latent_context_update_count=scalar_or_tensor(denoise_steps),
+            average_commit_len=scalar_or_tensor(average_commit_len, floating=True),
+            state_shift_count=scalar_or_tensor(shifts),
+            latent_memory_norm=scalar_or_tensor(latent_memory_norm, floating=True),
+            state_retention_score=scalar_or_tensor(
+                state_retention_score,
+                floating=True,
+            ),
+        )
+
+
+__all__ = [
+    "ModilifyMk1GenerationConfig",
+    "ModilifyMk1GenerationMixin",
+    "ModilifyMk1GenerationOutput",
+]

latent_deliberation.py

@@ -0,0 +1,490 @@
+# Copyright 2026 Modilify
+# SPDX-License-Identifier: LicenseRef-Modilify-Open-Model-1.0
+"""Fixed-shape latent deliberation state for Modilify Mk1 decoding.
+
+The state contains no vocabulary-sized tensors. Per-canvas information stays in
+a compact latent space so iterative diffusion does not retain one logits
+allocation per denoise pass.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+import math
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+
+
+@dataclass
+class LatentDeliberationState:
+    """Persistent, fixed-size state for one or more canvas episodes."""
+
+    token_latents: torch.Tensor
+    memory_slots: torch.Tensor
+    confidence: torch.Tensor
+    entropy: torch.Tensor
+    age: torch.Tensor
+    token_changed: torch.Tensor
+    confidence_delta: torch.Tensor
+    entropy_delta: torch.Tensor
+    ponder_steps: torch.Tensor
+    stagnation_steps: torch.Tensor
+
+    @classmethod
+    def empty(
+        cls,
+        *,
+        batch_size: int,
+        canvas_length: int,
+        latent_dim: int,
+        memory_slots: int,
+        device: torch.device,
+        dtype: torch.dtype,
+    ) -> "LatentDeliberationState":
+        """Create a zero-initialized recurrent state.
+
+        Args:
+            batch_size: Number of independent sequences.
+            canvas_length: Number of rolling canvas positions.
+            latent_dim: Width of each latent token and memory slot.
+            memory_slots: Number of persistent memory slots.
+            device: Allocation device.
+            dtype: Floating-point dtype for latent tensors.
+
+        Returns:
+            A zero-initialized state with integer progress clocks.
+        """
+
+        return cls(
+            token_latents=torch.zeros(
+                batch_size, canvas_length, latent_dim, device=device, dtype=dtype
+            ),
+            memory_slots=torch.zeros(
+                batch_size, memory_slots, latent_dim, device=device, dtype=dtype
+            ),
+            confidence=torch.zeros(
+                batch_size, canvas_length, device=device, dtype=torch.float32
+            ),
+            entropy=torch.zeros(
+                batch_size, canvas_length, device=device, dtype=torch.float32
+            ),
+            age=torch.zeros(
+                batch_size, canvas_length, device=device, dtype=torch.int32
+            ),
+            token_changed=torch.zeros(
+                batch_size, canvas_length, device=device, dtype=torch.float32
+            ),
+            confidence_delta=torch.zeros(
+                batch_size, canvas_length, device=device, dtype=torch.float32
+            ),
+            entropy_delta=torch.zeros(
+                batch_size, canvas_length, device=device, dtype=torch.float32
+            ),
+            ponder_steps=torch.zeros(batch_size, device=device, dtype=torch.int32),
+            stagnation_steps=torch.zeros(batch_size, device=device, dtype=torch.int32),
+        )
+
+    def shift(
+        self, committed: int, *, entropy_fill_value: float = 0.0
+    ) -> "LatentDeliberationState":
+        """Drop committed canvas positions without changing long-term memory.
+
+        Args:
+            committed: Number of leading canvas positions to drop.
+            entropy_fill_value: Fill value for newly exposed entropy slots.
+
+        Returns:
+            A shifted state with unchanged memory slots.
+        """
+
+        canvas_length = self.token_latents.shape[1]
+        if not 0 <= committed <= canvas_length:
+            raise ValueError("`committed` must be in [0, canvas_length].")
+        if committed == 0:
+            return self
+
+        def shifted(tensor: torch.Tensor, fill_value: float | int = 0) -> torch.Tensor:
+            result = torch.full_like(tensor, fill_value)
+            if committed < canvas_length:
+                result[:, : canvas_length - committed] = tensor[:, committed:]
+            return result
+
+        return LatentDeliberationState(
+            token_latents=shifted(self.token_latents),
+            memory_slots=self.memory_slots.clone(),
+            confidence=shifted(self.confidence),
+            entropy=shifted(self.entropy, entropy_fill_value),
+            age=shifted(self.age),
+            token_changed=shifted(self.token_changed),
+            confidence_delta=shifted(self.confidence_delta),
+            entropy_delta=shifted(self.entropy_delta),
+            ponder_steps=torch.zeros_like(self.ponder_steps),
+            stagnation_steps=torch.zeros_like(self.stagnation_steps),
+        )
+
+
+def advance_trajectory_clocks(
+    ponder_steps: torch.Tensor,
+    stagnation_steps: torch.Tensor,
+    *,
+    commit_lengths: torch.LongTensor,
+    active_rows: torch.BoolTensor,
+    progress_scores: torch.Tensor,
+    min_progress: float,
+) -> tuple[torch.IntTensor, torch.IntTensor]:
+    """Advance useful-ponder and true-stagnation clocks for each row.
+
+    Args:
+        ponder_steps: Current useful-ponder clocks, shape ``[batch]``.
+        stagnation_steps: Current stagnation clocks, shape ``[batch]``.
+        commit_lengths: Tokens committed this step.
+        active_rows: Rows that are still generating.
+        progress_scores: Signed fused-risk improvement.
+        min_progress: Minimum improvement counted as progress.
+
+    Returns:
+        Updated ponder and stagnation clocks.
+    """
+
+    if min_progress < 0:
+        raise ValueError("`min_progress` must be non-negative.")
+    if not (
+        ponder_steps.shape
+        == stagnation_steps.shape
+        == commit_lengths.shape
+        == active_rows.shape
+        == progress_scores.shape
+    ):
+        raise ValueError("Trajectory clock inputs must share shape [batch].")
+    committed = commit_lengths.gt(0)
+    waiting = active_rows & ~committed
+    improving = progress_scores.ge(min_progress)
+    next_ponder = torch.where(
+        committed, torch.zeros_like(ponder_steps), ponder_steps + waiting.to(torch.int32)
+    )
+    next_stagnation = torch.where(
+        committed,
+        torch.zeros_like(stagnation_steps),
+        torch.where(
+            waiting & improving,
+            torch.zeros_like(stagnation_steps),
+            stagnation_steps + waiting.to(torch.int32),
+        ),
+    )
+    return next_ponder.to(torch.int32), next_stagnation.to(torch.int32)
+
+
+def should_force_trajectory_jump(
+    ponder_steps: torch.Tensor,
+    stagnation_steps: torch.Tensor,
+    *,
+    max_ponder_steps: int,
+    stagnation_threshold: int,
+) -> torch.BoolTensor:
+    """Return whether a row has exhausted ponder or stagnation budget."""
+
+    if max_ponder_steps <= 0 or stagnation_threshold <= 0:
+        raise ValueError("Trajectory jump limits must be positive.")
+    return ponder_steps.ge(max_ponder_steps) | stagnation_steps.ge(stagnation_threshold)
+
+
+class _TemporalTransformerCell(nn.Module):
+    """One-step recurrent token update with fixed-slot memory attention."""
+
+    def __init__(
+        self,
+        latent_dim: int,
+        num_heads: int,
+        dropout: float,
+        local_attention_window: int,
+    ) -> None:
+        super().__init__()
+        self.state_norm = nn.LayerNorm(latent_dim)
+        self.observation_norm = nn.LayerNorm(latent_dim)
+        self.memory_address_norm = nn.LayerNorm(latent_dim)
+        self.memory_value_norm = nn.LayerNorm(latent_dim)
+        self.temporal_update = nn.Linear(2 * latent_dim, 2 * latent_dim)
+        self.local_attention = nn.MultiheadAttention(
+            latent_dim, num_heads, dropout=dropout, batch_first=True
+        )
+        self.local_attention_window = local_attention_window
+        self.register_buffer("_local_attention_mask", torch.empty(0), persistent=False)
+        self.token_memory_attention = nn.MultiheadAttention(
+            latent_dim, num_heads, dropout=dropout, batch_first=True
+        )
+        self.memory_token_attention = nn.MultiheadAttention(
+            latent_dim, num_heads, dropout=dropout, batch_first=True
+        )
+        self.token_ff_norm = nn.LayerNorm(latent_dim)
+        self.memory_ff_norm = nn.LayerNorm(latent_dim)
+        self.stored_token_norm = nn.LayerNorm(latent_dim)
+        self.stored_memory_norm = nn.LayerNorm(latent_dim)
+        expansion = latent_dim * 4
+        self.token_ff = nn.Sequential(
+            nn.Linear(latent_dim, expansion),
+            nn.SiLU(),
+            nn.Linear(expansion, latent_dim),
+        )
+        self.memory_ff = nn.Sequential(
+            nn.Linear(latent_dim, expansion),
+            nn.SiLU(),
+            nn.Linear(expansion, latent_dim),
+        )
+
+    def forward(
+        self,
+        previous_tokens: torch.Tensor,
+        observation: torch.Tensor,
+        memory: torch.Tensor,
+        memory_slot_identity: torch.Tensor,
+    ) -> tuple[torch.Tensor, torch.Tensor]:
+        """Update token latents and persistent memory for one cell.
+
+        Slot identity is an addressing key only. After the memory residual is
+        normalized, the same identity is added back so persistent content stays
+        slot-specific and invariant to canvas commits.
+
+        Args:
+            previous_tokens: Previous token latents, shape ``[batch, canvas, dim]``.
+            observation: Current observation, same shape as ``previous_tokens``.
+            memory: Persistent memory slots, shape ``[batch, slots, dim]``.
+            memory_slot_identity: Scaled slot addresses, same shape as ``memory``.
+
+        Returns:
+            Updated token latents and memory slots.
+        """
+
+        gate_logits, candidate = self.temporal_update(
+            torch.cat(
+                (self.state_norm(previous_tokens), self.observation_norm(observation)),
+                dim=-1,
+            )
+        ).chunk(2, dim=-1)
+        gate = torch.sigmoid(gate_logits)
+        tokens = gate * previous_tokens + (1.0 - gate) * torch.nn.functional.silu(candidate)
+        if (
+            self._local_attention_mask.shape != (tokens.shape[1], tokens.shape[1])
+            or self._local_attention_mask.device != tokens.device
+            or self._local_attention_mask.dtype != tokens.dtype
+        ):
+            positions = torch.arange(tokens.shape[1], device=tokens.device)
+            allowed = (positions[:, None] - positions[None, :]).abs() < self.local_attention_window
+            self._local_attention_mask = torch.zeros(
+                tokens.shape[1], tokens.shape[1], device=tokens.device, dtype=tokens.dtype
+            ).masked_fill(~allowed, torch.finfo(tokens.dtype).min)
+        local_update, _ = self.local_attention(
+            self.state_norm(tokens),
+            self.state_norm(tokens),
+            self.state_norm(tokens),
+            attn_mask=self._local_attention_mask,
+            need_weights=False,
+        )
+        tokens = tokens + local_update
+
+        addressed_memory = self.memory_address_norm(memory + memory_slot_identity)
+        memory_values = self.memory_value_norm(memory)
+        token_memory_update, _ = self.token_memory_attention(
+            self.state_norm(tokens), addressed_memory, memory_values, need_weights=False
+        )
+        tokens = tokens + token_memory_update
+        tokens = tokens + self.token_ff(self.token_ff_norm(tokens))
+
+        memory_token_update, _ = self.memory_token_attention(
+            addressed_memory,
+            self.state_norm(tokens),
+            self.state_norm(tokens),
+            need_weights=False,
+        )
+        memory = memory + memory_token_update
+        memory = memory + self.memory_ff(self.memory_ff_norm(memory))
+        return (
+            self.stored_token_norm(tokens),
+            self.stored_memory_norm(memory) + memory_slot_identity,
+        )
+
+
+class LatentDeliberationTransformer(nn.Module):
+    """Small recurrent Transformer that compresses repeated denoise context."""
+
+    def __init__(
+        self,
+        *,
+        hidden_size: int,
+        latent_dim: int = 512,
+        memory_slots: int = 16,
+        num_layers: int = 2,
+        num_heads: int = 8,
+        local_attention_window: int = 32,
+        dropout: float = 0.0,
+    ) -> None:
+        super().__init__()
+        if latent_dim % num_heads:
+            raise ValueError("`latent_dim` must be divisible by `num_heads`.")
+        if local_attention_window <= 0:
+            raise ValueError("`local_attention_window` must be positive.")
+        self.hidden_size = hidden_size
+        self.latent_dim = latent_dim
+        self.memory_slots = memory_slots
+        self.heavy_projection = nn.Linear(hidden_size, latent_dim, bias=False)
+        self.embedding_projection = nn.Linear(hidden_size, latent_dim, bias=False)
+        self.scalar_projection = nn.Linear(11, latent_dim, bias=False)
+        self.blocks = nn.ModuleList(
+            [
+                _TemporalTransformerCell(
+                    latent_dim, num_heads, dropout, local_attention_window
+                )
+                for _ in range(num_layers)
+            ]
+        )
+        self.output_norm = nn.LayerNorm(latent_dim)
+        self.output_projection = nn.Linear(latent_dim, hidden_size, bias=False)
+        self.memory_slot_identity = nn.Parameter(torch.empty(memory_slots, latent_dim))
+        self.reset_memory_slot_identity()
+
+    @torch.no_grad()
+    def reset_memory_slot_identity(self) -> None:
+        """Restore orthonormal slot addresses after generic initialization."""
+
+        workspace = torch.empty_like(self.memory_slot_identity, dtype=torch.float32)
+        if self.memory_slots <= self.latent_dim:
+            nn.init.orthogonal_(workspace)
+        else:
+            nn.init.normal_(workspace, mean=0.0, std=1.0)
+            workspace = F.normalize(workspace, dim=-1)
+        self.memory_slot_identity.copy_(workspace.to(dtype=self.memory_slot_identity.dtype))
+
+    def scaled_memory_slot_identity(
+        self,
+        *,
+        batch_size: int,
+        device: torch.device,
+        dtype: torch.dtype,
+    ) -> torch.Tensor:
+        """Return unit directions scaled to LayerNorm RMS.
+
+        Args:
+            batch_size: Number of sequences to broadcast over.
+            device: Output device.
+            dtype: Output dtype.
+
+        Returns:
+            Slot identities of shape ``[batch, slots, latent_dim]``.
+        """
+
+        identity = F.normalize(self.memory_slot_identity.float(), dim=-1)
+        identity = identity * math.sqrt(self.latent_dim)
+        return identity.to(device=device, dtype=dtype).unsqueeze(0).expand(
+            batch_size, -1, -1
+        )
+
+    def project_context(self, token_latents: torch.Tensor) -> torch.Tensor:
+        """Translate latent state into a self-conditioning embedding."""
+
+        return self.output_projection(self.output_norm(token_latents))
+
+    def forward(
+        self,
+        *,
+        heavy_hidden: torch.Tensor,
+        token_embeddings: torch.Tensor,
+        confidence: torch.Tensor,
+        entropy: torch.Tensor,
+        state: LatentDeliberationState,
+    ) -> tuple[torch.Tensor, LatentDeliberationState]:
+        """Advance latent memory and produce decoder self-conditioning.
+
+        Args:
+            heavy_hidden: Hidden states from the previous decoder pass.
+            token_embeddings: Embeddings of current noisy canvas tokens.
+            confidence: Proposal confidence for each canvas position.
+            entropy: Proposal entropy for each canvas position.
+            state: Persistent latent state from the preceding pass.
+
+        Returns:
+            Self-conditioning embeddings and the next compact latent state.
+        """
+
+        if heavy_hidden.ndim != 3:
+            raise ValueError("`heavy_hidden` must have shape [batch, canvas, hidden].")
+        if heavy_hidden.shape != token_embeddings.shape:
+            raise ValueError("`heavy_hidden` and `token_embeddings` must have the same shape.")
+        batch_size, canvas_length, hidden_size = heavy_hidden.shape
+        if hidden_size != self.hidden_size:
+            raise ValueError("Unexpected hidden size for latent deliberation.")
+        expected_state = (batch_size, canvas_length, self.latent_dim)
+        if state.token_latents.shape != expected_state:
+            raise ValueError("State token latents do not match the current canvas.")
+        if state.memory_slots.shape != (batch_size, self.memory_slots, self.latent_dim):
+            raise ValueError("State memory slots do not match this module.")
+        if state.age.dtype is not torch.int32:
+            raise TypeError("Latent deliberation ages must use int32.")
+
+        scalars = torch.stack(
+            (
+                confidence.to(dtype=heavy_hidden.dtype),
+                entropy.to(dtype=heavy_hidden.dtype).log1p(),
+                state.age.to(dtype=heavy_hidden.dtype).clamp_max(32767).log1p(),
+                torch.linspace(
+                    -1.0,
+                    1.0,
+                    canvas_length,
+                    device=heavy_hidden.device,
+                    dtype=heavy_hidden.dtype,
+                )
+                .unsqueeze(0)
+                .expand(batch_size, -1),
+                state.token_changed.to(dtype=heavy_hidden.dtype),
+                state.confidence_delta.to(dtype=heavy_hidden.dtype),
+                state.entropy_delta.to(dtype=heavy_hidden.dtype).sign()
+                * state.entropy_delta.to(dtype=heavy_hidden.dtype).abs().log1p(),
+                state.ponder_steps.to(dtype=heavy_hidden.dtype).log1p()[:, None].expand(
+                    -1, canvas_length
+                ),
+                state.stagnation_steps.to(dtype=heavy_hidden.dtype).log1p()[:, None].expand(
+                    -1, canvas_length
+                ),
+                confidence.to(dtype=heavy_hidden.dtype)
+                * torch.exp(-entropy.to(dtype=heavy_hidden.dtype).clamp_min(0.0)),
+                state.confidence_delta.to(dtype=heavy_hidden.dtype).clamp_min(0.0)
+                + (-state.entropy_delta.to(dtype=heavy_hidden.dtype)).clamp_min(0.0).log1p(),
+            ),
+            dim=-1,
+        )
+        observation = (
+            self.heavy_projection(heavy_hidden)
+            + self.embedding_projection(token_embeddings)
+            + self.scalar_projection(scalars)
+        )
+        tokens = state.token_latents
+        memory = state.memory_slots
+        slot_identity = self.scaled_memory_slot_identity(
+            batch_size=batch_size,
+            device=memory.device,
+            dtype=memory.dtype,
+        )
+        for block in self.blocks:
+            tokens, memory = block(tokens, observation, memory, slot_identity)
+            observation = tokens
+        next_state = LatentDeliberationState(
+            token_latents=tokens,
+            memory_slots=memory,
+            confidence=confidence.to(dtype=torch.float32),
+            entropy=entropy.to(dtype=torch.float32),
+            age=state.age,
+            token_changed=state.token_changed,
+            confidence_delta=state.confidence_delta,
+            entropy_delta=state.entropy_delta,
+            ponder_steps=state.ponder_steps,
+            stagnation_steps=state.stagnation_steps,
+        )
+        return self.project_context(tokens), next_state
+
+
+__all__ = [
+    "LatentDeliberationState",
+    "LatentDeliberationTransformer",
+    "advance_trajectory_clocks",
+    "should_force_trajectory_jump",
+]

model-00001-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:990b322d47df486108e725ff6884842b4b6512e98e57202d04be8b57ecd2b456
+size 5363460572

model-00002-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b2b7137b04fad66c8654441a9535b257862ee9f97ddf9d4b635ac3fe09b6366d
+size 4884577974

model-00003-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c58d6983f7796596da51974362aefc6f578aea6c572eb1ff404c76d10ae6d69
+size 4913414718

model-00004-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:89ff79354e9cbb2912756be80fd9dfc487fdb226f25e78248397080c89f3d830
+size 4884577998

model-00005-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41a2615fab0008f86cd27ea11d04edda0cea8f8e92b3f579ee2f21c4fc642357
+size 4913414782

model-00006-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8059cb7856f4bc999bfbba2ef805bbfca7caa4857f07388f75369064cfa01209
+size 4884578038

model-00007-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e5e17d862209d68ef619895f1ac31b1dded2a3c151738e8241d9c85d147c764
+size 4913414782

model-00008-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f553a5906c249c4c2181fed3fa6b59ea6b8e8bfe6a0844b5a375c7e81957edd
+size 4884578038

model-00009-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46bdb5a161862beeebe862d5dba46562b8d0c511c84aac87355433b45a30ae4c
+size 4913414782

model-00010-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1d6491992dfe7b4ccf4ee266594bc754c90dace3c8c7fb21e5c149e64b57841
+size 4884578038

model-00011-of-00011.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b60ca12117cdb0165501adf18ecb7fef85310db6565cf6004bc72aebfe726573
+size 2838371094

modeling_modilify_mk1.py

@@ -0,0 +1,688 @@
+# Copyright 2026 Modilify
+# SPDX-License-Identifier: LicenseRef-Modilify-Open-Model-1.0
+"""Standard PyTorch multimodal model implementation for Modilify Mk1."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from dataclasses import dataclass, replace
+import math
+from typing import Any
+
+import torch
+from torch import nn
+from torch.nn import functional as F
+from transformers.cache_utils import Cache
+from transformers.masking_utils import (
+    ALL_MASK_ATTENTION_FUNCTIONS,
+    bidirectional_mask_function,
+)
+from transformers.modeling_outputs import BaseModelOutputWithPast
+from transformers.utils import ModelOutput
+from transformers.models.diffusion_gemma import (
+    DiffusionGemmaDecoderModel,
+    DiffusionGemmaEncoderModel,
+    DiffusionGemmaPreTrainedModel,
+)
+from transformers.models.diffusion_gemma.modeling_diffusion_gemma import (
+    DiffusionGemmaRMSNorm,
+    DiffusionGemmaTextRouter,
+)
+
+from .configuration_modilify_mk1 import ModilifyMk1Config
+from .generation_modilify_mk1 import (
+    ModilifyMk1GenerationConfig,
+    ModilifyMk1GenerationMixin,
+)
+from .latent_deliberation import (
+    LatentDeliberationState,
+    LatentDeliberationTransformer,
+)
+
+
+@dataclass
+class ModilifyMk1DecoderOutput(BaseModelOutputWithPast):
+    """Decoder hidden states and latent-context diagnostics."""
+
+    token_embeddings: torch.FloatTensor | None = None
+    latent_residual_diagnostics: dict[str, torch.Tensor] | None = None
+
+
+@dataclass
+class ModilifyMk1ModelOutput(BaseModelOutputWithPast):
+    """Combined multimodal encoder and diffusion decoder output."""
+
+    token_embeddings: torch.FloatTensor | None = None
+    encoder_last_hidden_state: torch.FloatTensor | None = None
+    latent_residual_diagnostics: dict[str, torch.Tensor] | None = None
+
+
+@dataclass
+class ModilifyMk1BlockDiffusionOutput(ModelOutput):
+    """Inference output used by the rolling diffusion generator."""
+
+    logits: torch.FloatTensor | None = None
+    heavy_hidden_state: torch.FloatTensor | None = None
+    next_latent_state: LatentDeliberationState | None = None
+    past_key_values: Cache | None = None
+    encoder_last_hidden_state: torch.FloatTensor | None = None
+    temporal_context: torch.FloatTensor | None = None
+    latent_residual_diagnostics: dict[str, torch.Tensor] | None = None
+    proposal: torch.LongTensor | None = None
+    proposal_confidence: torch.FloatTensor | None = None
+    token_entropy: torch.FloatTensor | None = None
+    greedy_proposal: torch.LongTensor | None = None
+    greedy_confidence: torch.FloatTensor | None = None
+
+
+class ModilifyMk1RMSNorm(DiffusionGemmaRMSNorm):
+    """Official RMSNorm parameters with a same-dtype residual forward."""
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        """Normalize ``hidden_states`` and restore the input dtype."""
+
+        normed_output = self._norm(hidden_states)
+        if self.with_scale:
+            normed_output = normed_output * self.weight.to(dtype=normed_output.dtype)
+        return normed_output.type_as(hidden_states)
+
+
+class ModilifyMk1TextRouter(DiffusionGemmaTextRouter):
+    """Official router parameters with a log-softmax top-k route."""
+
+    def __init__(self, config: Any) -> None:
+        super().__init__(config)
+        self.norm = ModilifyMk1RMSNorm(self.hidden_size, eps=self.eps, with_scale=False)
+
+    def forward(
+        self, hidden_states: torch.Tensor
+    ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+        """Return route probabilities, top-k weights, and expert indices."""
+
+        hidden_states = self.norm(hidden_states)
+        hidden_states = hidden_states * self.scale * self.scalar_root_size
+        expert_scores = self.proj(hidden_states)
+        router_probabilities = F.log_softmax(expert_scores, dim=-1).exp()
+        top_k_weights, top_k_index = torch.topk(
+            router_probabilities,
+            k=self.config.top_k_experts,
+            dim=-1,
+        )
+        top_k_weights = top_k_weights / top_k_weights.sum(dim=-1, keepdim=True)
+        top_k_weights = top_k_weights * self.per_expert_scale[top_k_index]
+        return router_probabilities, top_k_weights, top_k_index
+
+
+def install_modilify_mk1_trunk_semantics(module: nn.Module) -> None:
+    """Replace official leaf modules on this instance only.
+
+    Args:
+        module: Encoder, decoder, or parent module whose children should be
+            swapped to the instance-scoped RMSNorm and router implementations.
+    """
+
+    for name, child in list(module.named_children()):
+        if type(child) is DiffusionGemmaRMSNorm:
+            dim = int(child.weight.shape[0]) if child.with_scale else 1
+            replacement = ModilifyMk1RMSNorm(
+                dim, eps=child.eps, with_scale=child.with_scale
+            )
+            replacement.load_state_dict(child.state_dict())
+            setattr(module, name, replacement)
+        elif type(child) is DiffusionGemmaTextRouter:
+            replacement = ModilifyMk1TextRouter(child.config)
+            replacement.load_state_dict(child.state_dict())
+            setattr(module, name, replacement)
+        else:
+            install_modilify_mk1_trunk_semantics(child)
+
+
+class ModilifyMk1EncoderModel(DiffusionGemmaEncoderModel):
+    """Unmodified Transformers DiffusionGemma multimodal encoder."""
+
+    config_class = ModilifyMk1Config
+
+
+class ModilifyMk1DecoderModel(DiffusionGemmaDecoderModel):
+    """DiffusionGemma decoder conditioned by recurrent latent embeddings."""
+
+    config_class = ModilifyMk1Config
+    latent_residual_rms_ratio_cap = 0.5
+
+    @staticmethod
+    def create_diffusion_decoder_attention_mask(
+        config: Any,
+        inputs_embeds: torch.Tensor,
+        past_key_values: Cache,
+        decoder_attention_mask: torch.Tensor | dict | None = None,
+    ) -> dict[str, torch.Tensor | None]:
+        """Build bidirectional canvas masks without skipping sliding layers.
+
+        Args:
+            config: Text configuration used for layer types and window size.
+            inputs_embeds: Canvas embeddings that define query length and dtype.
+            past_key_values: Prefix cache used to size the key/value axis.
+            decoder_attention_mask: Optional 2-D mask or precomputed 4-D maps.
+
+        Returns:
+            A mapping from layer pattern to attention mask.
+        """
+
+        if past_key_values is None:
+            raise ValueError(
+                "The diffusion mask requires `past_key_values` to construct the "
+                "next attention mask correctly."
+            )
+        if (
+            decoder_attention_mask is None
+            or config._attn_implementation
+            not in ALL_MASK_ATTENTION_FUNCTIONS._global_mapping
+        ):
+            return {"full_attention": None, "sliding_attention": None}
+        if isinstance(decoder_attention_mask, dict) and all(
+            mask.ndim == 4 for mask in decoder_attention_mask.values()
+        ):
+            return decoder_attention_mask
+
+        text_config = config.get_text_config() if hasattr(config, "get_text_config") else config
+        q_length = inputs_embeds.shape[1]
+        q_offset = past_key_values.get_seq_length()
+        if isinstance(q_offset, torch.Tensor):
+            q_offset = q_offset.to(inputs_embeds.device)
+        additional_kv_length = (
+            getattr(config, "canvas_length", 0) if past_key_values.is_compileable else 0
+        )
+        mask_mapping: dict[str, torch.Tensor | None] = {}
+        for layer_pattern in set(text_config.layer_types):
+            layer_idx = past_key_values.is_sliding.index(
+                layer_pattern == "sliding_attention"
+            )
+            kv_length, kv_offset = past_key_values.get_mask_sizes(q_length, layer_idx)
+            kv_length += additional_kv_length
+            if layer_pattern == "sliding_attention" and past_key_values.is_compileable:
+                sliding_layer = past_key_values.layers[layer_idx]
+                max_length = sliding_layer.get_max_length() + additional_kv_length
+                if kv_length >= max_length:
+                    kv_length = max_length
+            mask_mapping[layer_pattern] = ALL_MASK_ATTENTION_FUNCTIONS[
+                config._attn_implementation
+            ](
+                batch_size=inputs_embeds.shape[0],
+                q_length=q_length,
+                kv_length=kv_length,
+                q_offset=q_offset,
+                kv_offset=kv_offset,
+                mask_function=bidirectional_mask_function,
+                attention_mask=decoder_attention_mask,
+                allow_is_causal_skip=False,
+                allow_is_bidirectional_skip=True,
+                local_size=getattr(text_config, "sliding_window", None),
+                dtype=inputs_embeds.dtype,
+                config=text_config,
+                use_vmap=False,
+                device=inputs_embeds.device,
+            )
+        return mask_mapping
+
+    def merge_latent_context(
+        self,
+        token_embeddings: torch.Tensor,
+        latent_context: torch.Tensor | None,
+    ) -> tuple[torch.Tensor, dict[str, torch.Tensor]]:
+        """Apply the native self-conditioning bridge to latent context.
+
+        Args:
+            token_embeddings: Embedded noisy canvas tokens.
+            latent_context: Context emitted by the latent Transformer.
+
+        Returns:
+            Merged embeddings and scalar diagnostic tensors.
+        """
+
+        context = (
+            torch.zeros_like(token_embeddings)
+            if latent_context is None
+            else latent_context.to(token_embeddings)
+        )
+        if context.shape != token_embeddings.shape:
+            raise ValueError("Latent context must match the canvas embedding shape.")
+        mapper = self.self_conditioning
+        normalized = mapper.pre_norm(context)
+        mapped = mapper.down_proj(
+            mapper.act_fn(mapper.gate_proj(normalized)) * mapper.up_proj(normalized)
+        )
+        mapped_rms_per_token = mapped.float().square().mean(dim=-1, keepdim=True).sqrt()
+        token_rms_per_token = (
+            token_embeddings.float().square().mean(dim=-1, keepdim=True).sqrt()
+        )
+        cap = self.latent_residual_rms_ratio_cap * token_rms_per_token
+        scale = cap / torch.sqrt(mapped_rms_per_token.square() + cap.square() + 1.0e-12)
+        mapped = mapped * scale.to(mapped)
+        combined = mapper.post_norm(token_embeddings + mapped)
+        token_rms = token_embeddings.detach().float().square().mean().sqrt()
+        mapped_rms = mapped.detach().float().square().mean().sqrt()
+        diagnostics = {
+            "token_embedding_rms": token_rms,
+            "latent_context_rms": context.detach().float().square().mean().sqrt(),
+            "mapped_context_rms": mapped_rms,
+            "latent_to_embedding_rms_ratio": mapped_rms / token_rms.clamp_min(1.0e-12),
+        }
+        return combined, diagnostics
+
+    def forward(
+        self,
+        decoder_input_ids: torch.LongTensor,
+        past_key_values: Cache | None = None,
+        temporal_context_embeddings: torch.FloatTensor | None = None,
+        decoder_attention_mask: torch.Tensor | dict | None = None,
+        decoder_position_ids: torch.LongTensor | None = None,
+        **kwargs: Any,
+    ) -> ModilifyMk1DecoderOutput:
+        """Decode one noisy canvas using Transformers and PyTorch operations."""
+
+        token_embeddings = self.embed_tokens(decoder_input_ids)
+        inputs_embeds, diagnostics = self.merge_latent_context(
+            token_embeddings,
+            temporal_context_embeddings,
+        )
+        if decoder_position_ids is None:
+            prefix = past_key_values.get_seq_length(0) if past_key_values is not None else 0
+            decoder_position_ids = torch.arange(
+                prefix,
+                prefix + inputs_embeds.shape[1],
+                device=inputs_embeds.device,
+            ).unsqueeze(0)
+        if not isinstance(mask_mapping := decoder_attention_mask, dict):
+            mask_mapping = self.create_diffusion_decoder_attention_mask(
+                config=self.text_config,
+                inputs_embeds=inputs_embeds,
+                past_key_values=past_key_values,
+                decoder_attention_mask=decoder_attention_mask,
+            )
+        position_embeddings = {
+            layer_type: self.rotary_emb(inputs_embeds, decoder_position_ids, layer_type)
+            for layer_type in self.unique_layer_types
+        }
+        hidden_states = inputs_embeds
+        for index, layer in enumerate(self.layers[: self.text_config.num_hidden_layers]):
+            layer_type = self.text_config.layer_types[index]
+            hidden_states = layer(
+                hidden_states,
+                position_embeddings=position_embeddings[layer_type],
+                attention_mask=mask_mapping[layer_type],
+                position_ids=decoder_position_ids,
+                past_key_values=past_key_values,
+                **kwargs,
+            )
+        return ModilifyMk1DecoderOutput(
+            last_hidden_state=self.norm(hidden_states),
+            past_key_values=past_key_values,
+            token_embeddings=token_embeddings,
+            latent_residual_diagnostics=diagnostics,
+        )
+
+
+class ModilifyMk1Model(DiffusionGemmaPreTrainedModel):
+    """Multimodal encoder plus latent-conditioned block diffusion decoder."""
+
+    config_class = ModilifyMk1Config
+    _tied_weights_keys = {
+        "encoder.language_model.norm.weight": "decoder.norm.weight",
+        r"encoder.language_model.layers\.(?:[^.]+\.)*weight": r"decoder.layers\.(?:[^.]+\.)*weight",
+        r"encoder.language_model.layers\.(?:[^.]+\.)*scale": r"decoder.layers\.(?:[^.]+\.)*scale",
+        (
+            r"encoder.language_model.layers\.(?:[^.]+\.)*per_expert_scale"
+        ): r"decoder.layers\.(?:[^.]+\.)*per_expert_scale",
+        (
+            r"encoder.language_model.layers\.(?:[^.]+\.)*gate_up_proj"
+        ): r"decoder.layers\.(?:[^.]+\.)*gate_up_proj",
+        (
+            r"encoder.language_model.layers\.(?:[^.]+\.)*down_proj"
+        ): r"decoder.layers\.(?:[^.]+\.)*down_proj",
+        "encoder.language_model.embed_tokens.weight": "decoder.embed_tokens.weight",
+    }
+
+    def __init__(self, config: ModilifyMk1Config) -> None:
+        super().__init__(config)
+        self.encoder = ModilifyMk1EncoderModel(config)
+        self.decoder = ModilifyMk1DecoderModel(config)
+        install_modilify_mk1_trunk_semantics(self)
+        self.post_init()
+
+    def get_encoder(self) -> ModilifyMk1EncoderModel:
+        """Return the multimodal encoder."""
+
+        return self.encoder
+
+    def get_decoder(self) -> ModilifyMk1DecoderModel:
+        """Return the diffusion decoder."""
+
+        return self.decoder
+
+    def get_input_embeddings(self) -> nn.Module:
+        """Return the shared text embedding module."""
+
+        return self.encoder.get_input_embeddings()
+
+    def set_input_embeddings(self, value: nn.Module) -> None:
+        """Set the shared text embedding module."""
+
+        self.encoder.set_input_embeddings(value)
+        self.decoder.embed_tokens = value
+
+    def forward(
+        self,
+        *,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | dict | None = None,
+        past_key_values: Cache | None = None,
+        position_ids: torch.LongTensor | None = None,
+        decoder_input_ids: torch.LongTensor,
+        temporal_context_embeddings: torch.FloatTensor | None = None,
+        decoder_attention_mask: torch.Tensor | dict | None = None,
+        decoder_position_ids: torch.LongTensor | None = None,
+        **kwargs: Any,
+    ) -> ModilifyMk1ModelOutput:
+        """Encode multimodal context and decode one canvas."""
+
+        encoder_hidden_state = None
+        encoder_keys = ("pixel_values", "mm_token_type_ids", "image_position_ids", "inputs_embeds")
+        encoder_kwargs = {key: kwargs.pop(key) for key in encoder_keys if key in kwargs}
+        if input_ids is not None:
+            encoded = self.encoder(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                past_key_values=past_key_values,
+                position_ids=position_ids,
+                **encoder_kwargs,
+            )
+            past_key_values = encoded.past_key_values
+            encoder_hidden_state = encoded.last_hidden_state
+        elif past_key_values is None:
+            raise ValueError("Either `input_ids` or `past_key_values` is required.")
+        decoded = self.decoder(
+            decoder_input_ids=decoder_input_ids,
+            past_key_values=past_key_values,
+            temporal_context_embeddings=temporal_context_embeddings,
+            decoder_attention_mask=decoder_attention_mask,
+            decoder_position_ids=decoder_position_ids,
+            **kwargs,
+        )
+        return ModilifyMk1ModelOutput(
+            last_hidden_state=decoded.last_hidden_state,
+            past_key_values=past_key_values,
+            token_embeddings=decoded.token_embeddings,
+            encoder_last_hidden_state=encoder_hidden_state,
+            latent_residual_diagnostics=decoded.latent_residual_diagnostics,
+        )
+
+
+class ModilifyMk1ForBlockDiffusion(
+    DiffusionGemmaPreTrainedModel,
+    ModilifyMk1GenerationMixin,
+):
+    """Inference-only multimodal Modilify Mk1 model."""
+
+    config_class = ModilifyMk1Config
+    _tied_weights_keys = {"lm_head.weight": "model.decoder.embed_tokens.weight"}
+    generation_config_class = ModilifyMk1GenerationConfig
+
+    @torch.no_grad()
+    def _init_weights(self, module: nn.Module) -> None:
+        super()._init_weights(module)
+        if isinstance(module, LatentDeliberationTransformer):
+            module.reset_memory_slot_identity()
+
+    def __init__(self, config: ModilifyMk1Config) -> None:
+        super().__init__(config)
+        self.model = ModilifyMk1Model(config)
+        self.latent_deliberation = LatentDeliberationTransformer(
+            hidden_size=config.text_config.hidden_size,
+            latent_dim=config.latent_dim,
+            memory_slots=config.latent_memory_slots,
+            num_layers=config.latent_num_layers,
+            num_heads=config.latent_num_heads,
+            local_attention_window=config.latent_local_attention_window,
+            dropout=config.latent_dropout,
+        )
+        self.lm_head = nn.Linear(
+            config.text_config.hidden_size,
+            config.text_config.vocab_size,
+            bias=False,
+        )
+        self.final_logit_softcapping = config.text_config.final_logit_softcapping
+        self.post_init()
+
+    def _prepare_latent_context(
+        self,
+        decoder_input_ids: torch.LongTensor,
+        *,
+        history_hidden_state: torch.Tensor | None,
+        confidence: torch.Tensor | None,
+        entropy: torch.Tensor | None,
+        age: torch.Tensor | None,
+        latent_state: LatentDeliberationState | None,
+    ) -> tuple[torch.Tensor, LatentDeliberationState]:
+        """Advance recurrent latent state for the current canvas."""
+
+        batch_size, canvas_length = decoder_input_ids.shape
+        dtype = self.model.decoder.embed_tokens.weight.dtype
+        if latent_state is None:
+            latent_state = LatentDeliberationState.empty(
+                batch_size=batch_size,
+                canvas_length=canvas_length,
+                latent_dim=self.config.latent_dim,
+                memory_slots=self.config.latent_memory_slots,
+                device=decoder_input_ids.device,
+                dtype=dtype,
+            )
+        confidence = (
+            latent_state.confidence
+            if confidence is None
+            else confidence.squeeze(-1).float()
+        )
+        entropy = latent_state.entropy if entropy is None else entropy.squeeze(-1).float()
+        if age is not None:
+            latent_state = replace(
+                latent_state,
+                age=age.to(device=decoder_input_ids.device, dtype=torch.int32),
+            )
+        token_embeddings = self.model.decoder.embed_tokens(decoder_input_ids)
+        history = (
+            torch.zeros_like(token_embeddings)
+            if history_hidden_state is None
+            else history_hidden_state
+        )
+        return self.latent_deliberation(
+            heavy_hidden=history,
+            token_embeddings=token_embeddings,
+            confidence=confidence,
+            entropy=entropy,
+            state=latent_state,
+        )
+
+    def _apply_repetition_penalty(
+        self,
+        logits: torch.Tensor,
+        *,
+        repetition_token_mask: torch.BoolTensor | None,
+        repetition_penalty: float,
+    ) -> torch.Tensor:
+        """Apply a sign-aware Transformers repetition penalty.
+
+        Args:
+            logits: Soft-capped scores, shape ``[batch, canvas, vocab]``.
+            repetition_token_mask: Tokens already seen, shape ``[batch, vocab]``.
+            repetition_penalty: Penalty factor. ``1.0`` leaves logits unchanged.
+
+        Returns:
+            Penalized logits with the same shape as ``logits``.
+        """
+
+        if (
+            repetition_token_mask is None
+            or not math.isfinite(repetition_penalty)
+            or repetition_penalty == 1.0
+        ):
+            return logits
+        if repetition_penalty <= 0:
+            raise ValueError("`repetition_penalty` must be a positive finite number.")
+        if repetition_token_mask.shape != (logits.shape[0], logits.shape[-1]):
+            raise ValueError(
+                "`repetition_token_mask` must have shape [batch, vocab]."
+            )
+        scores = logits.float()
+        penalized = torch.where(scores < 0, scores * repetition_penalty, scores / repetition_penalty)
+        mask = repetition_token_mask.to(device=scores.device).unsqueeze(1)
+        return torch.where(mask, penalized, scores).to(dtype=logits.dtype)
+
+    def _proposal_statistics(
+        self,
+        logits: torch.Tensor,
+        *,
+        denoise_temperature: float | None = None,
+        repetition_token_mask: torch.BoolTensor | None = None,
+        repetition_penalty: float = 1.0,
+        sampling_generators: Sequence[torch.Generator] | None = None,
+    ) -> tuple[
+        torch.LongTensor,
+        torch.Tensor,
+        torch.Tensor,
+        torch.LongTensor,
+        torch.Tensor,
+    ]:
+        """Compute exact proposal statistics with standard PyTorch operations."""
+
+        temperature = (
+            self.config.denoise_temperature
+            if denoise_temperature is None
+            else float(denoise_temperature)
+        )
+        if not math.isfinite(temperature) or temperature <= 0.0:
+            raise ValueError("`denoise_temperature` must be positive.")
+        scores = self._apply_repetition_penalty(
+            logits,
+            repetition_token_mask=repetition_token_mask,
+            repetition_penalty=repetition_penalty,
+        ).float() / temperature
+        probabilities = torch.softmax(scores, dim=-1)
+        if sampling_generators is None:
+            proposal = torch.multinomial(
+                probabilities.reshape(-1, probabilities.shape[-1]),
+                num_samples=1,
+            ).view(logits.shape[:-1])
+        else:
+            if len(sampling_generators) != logits.shape[0]:
+                raise ValueError("Sampling requires one generator per batch row.")
+            rows = []
+            for row, generator in enumerate(sampling_generators):
+                rows.append(
+                    torch.multinomial(
+                        probabilities[row],
+                        num_samples=1,
+                        generator=generator,
+                    ).squeeze(-1)
+                )
+            proposal = torch.stack(rows, dim=0)
+        proposal_confidence = probabilities.gather(-1, proposal.unsqueeze(-1)).squeeze(-1)
+        greedy_proposal = probabilities.argmax(dim=-1)
+        greedy_confidence = probabilities.gather(
+            -1, greedy_proposal.unsqueeze(-1)
+        ).squeeze(-1)
+        token_entropy = -(
+            probabilities * probabilities.clamp_min(1.0e-30).log()
+        ).sum(dim=-1)
+        return (
+            proposal,
+            proposal_confidence,
+            token_entropy,
+            greedy_proposal,
+            greedy_confidence,
+        )
+
+    def forward(
+        self,
+        *,
+        input_ids: torch.LongTensor | None = None,
+        attention_mask: torch.Tensor | dict | None = None,
+        past_key_values: Cache | None = None,
+        position_ids: torch.LongTensor | None = None,
+        decoder_input_ids: torch.LongTensor,
+        previous_confidence: torch.FloatTensor | None = None,
+        previous_entropy: torch.FloatTensor | None = None,
+        token_age: torch.Tensor | None = None,
+        latent_state: LatentDeliberationState | None = None,
+        history_hidden_state: torch.FloatTensor | None = None,
+        decoder_attention_mask: torch.Tensor | dict | None = None,
+        decoder_position_ids: torch.LongTensor | None = None,
+        return_proposal_statistics: bool = False,
+        denoise_temperature: float | None = None,
+        repetition_token_mask: torch.BoolTensor | None = None,
+        repetition_penalty: float = 1.0,
+        sampling_generators: Sequence[torch.Generator] | None = None,
+        **kwargs: Any,
+    ) -> ModilifyMk1BlockDiffusionOutput:
+        """Run one inference step over a noisy diffusion canvas."""
+
+        latent_context, next_state = self._prepare_latent_context(
+            decoder_input_ids,
+            history_hidden_state=history_hidden_state,
+            confidence=previous_confidence,
+            entropy=previous_entropy,
+            age=token_age,
+            latent_state=latent_state,
+        )
+        outputs = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            position_ids=position_ids,
+            decoder_input_ids=decoder_input_ids,
+            temporal_context_embeddings=latent_context,
+            decoder_attention_mask=decoder_attention_mask,
+            decoder_position_ids=decoder_position_ids,
+            **kwargs,
+        )
+        logits = self.lm_head(outputs.last_hidden_state)
+        logits = (
+            torch.tanh(logits / self.final_logit_softcapping)
+            * self.final_logit_softcapping
+        )
+        statistics = (None, None, None, None, None)
+        if return_proposal_statistics:
+            statistics = self._proposal_statistics(
+                logits,
+                denoise_temperature=denoise_temperature,
+                repetition_token_mask=repetition_token_mask,
+                repetition_penalty=repetition_penalty,
+                sampling_generators=sampling_generators,
+            )
+        return ModilifyMk1BlockDiffusionOutput(
+            logits=None if return_proposal_statistics else logits,
+            heavy_hidden_state=outputs.last_hidden_state,
+            next_latent_state=next_state,
+            past_key_values=outputs.past_key_values,
+            encoder_last_hidden_state=outputs.encoder_last_hidden_state,
+            temporal_context=latent_context,
+            latent_residual_diagnostics=outputs.latent_residual_diagnostics,
+            proposal=statistics[0],
+            proposal_confidence=statistics[1],
+            token_entropy=statistics[2],
+            greedy_proposal=statistics[3],
+            greedy_confidence=statistics[4],
+        )
+
+
+ModilifyMk1Model.register_for_auto_class("AutoModel")
+ModilifyMk1ForBlockDiffusion.register_for_auto_class("AutoModelForCausalLM")
+ModilifyMk1ForBlockDiffusion.register_for_auto_class("AutoModelForMultimodalLM")
+
+
+__all__ = [
+    "ModilifyMk1BlockDiffusionOutput",
+    "ModilifyMk1Config",
+    "ModilifyMk1DecoderModel",
+    "ModilifyMk1EncoderModel",
+    "ModilifyMk1ForBlockDiffusion",
+    "ModilifyMk1Model",
+]

processor_config.json

@@ -0,0 +1,75 @@
+{
+  "audio_ms_per_token": 40,
+  "audio_seq_length": 750,
+  "feature_extractor": {
+    "dither": 0.0,
+    "feature_extractor_type": "Gemma4AudioFeatureExtractor",
+    "feature_size": 128,
+    "fft_length": 512,
+    "fft_overdrive": false,
+    "frame_length": 320,
+    "hop_length": 160,
+    "input_scale_factor": 1.0,
+    "max_frequency": 8000.0,
+    "mel_floor": 0.001,
+    "min_frequency": 0.0,
+    "padding_side": "right",
+    "padding_value": 0.0,
+    "per_bin_mean": null,
+    "per_bin_stddev": null,
+    "preemphasis": 0.0,
+    "preemphasis_htk_flavor": true,
+    "return_attention_mask": true,
+    "sampling_rate": 16000
+  },
+  "image_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": false,
+    "do_rescale": true,
+    "do_resize": true,
+    "image_mean": [
+      0.0,
+      0.0,
+      0.0
+    ],
+    "image_processor_type": "Gemma4ImageProcessor",
+    "image_seq_length": 280,
+    "image_std": [
+      1.0,
+      1.0,
+      1.0
+    ],
+    "max_soft_tokens": 280,
+    "patch_size": 16,
+    "pooling_kernel_size": 3,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098
+  },
+  "image_seq_length": 280,
+  "processor_class": "Gemma4Processor",
+  "video_processor": {
+    "do_convert_rgb": true,
+    "do_normalize": true,
+    "do_rescale": true,
+    "do_resize": true,
+    "do_sample_frames": true,
+    "image_mean": [
+      0.0,
+      0.0,
+      0.0
+    ],
+    "image_std": [
+      1.0,
+      1.0,
+      1.0
+    ],
+    "max_soft_tokens": 70,
+    "num_frames": 32,
+    "patch_size": 16,
+    "pooling_kernel_size": 3,
+    "resample": 3,
+    "rescale_factor": 0.00392156862745098,
+    "return_metadata": false,
+    "video_processor_type": "Gemma4VideoProcessor"
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc8d3a0ce36466ccc1278bf987df5f71db1719b9ca6b4118264f45cb627bfe0f
+size 32169626

tokenizer_config.json

@@ -0,0 +1,96 @@
+{
+  "audio_token": "<|audio|>",
+  "backend": "tokenizers",
+  "boa_token": "<|audio>",
+  "boi_token": "<|image>",
+  "bos_token": "<bos>",
+  "eoa_token": "<audio|>",
+  "eoc_token": "<channel|>",
+  "eoi_token": "<image|>",
+  "eos_token": "<eos>",
+  "eot_token": "<turn|>",
+  "escape_token": "<|\"|>",
+  "etc_token": "<tool_call|>",
+  "etd_token": "<tool|>",
+  "etr_token": "<tool_response|>",
+  "extra_special_tokens": [
+    "<|video|>"
+  ],
+  "image_token": "<|image|>",
+  "is_local": true,
+  "local_files_only": false,
+  "mask_token": "<mask>",
+  "model_max_length": 1000000000000000019884624838656,
+  "model_specific_special_tokens": {
+    "audio_token": "<|audio|>",
+    "boa_token": "<|audio>",
+    "boi_token": "<|image>",
+    "eoa_token": "<audio|>",
+    "eoc_token": "<channel|>",
+    "eoi_token": "<image|>",
+    "eot_token": "<turn|>",
+    "escape_token": "<|\"|>",
+    "etc_token": "<tool_call|>",
+    "etd_token": "<tool|>",
+    "etr_token": "<tool_response|>",
+    "image_token": "<|image|>",
+    "soc_token": "<|channel>",
+    "sot_token": "<|turn>",
+    "stc_token": "<|tool_call>",
+    "std_token": "<|tool>",
+    "str_token": "<|tool_response>",
+    "think_token": "<|think|>"
+  },
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "processor_class": "Gemma4Processor",
+  "response_schema": {
+    "properties": {
+      "content": {
+        "type": "string"
+      },
+      "role": {
+        "const": "assistant"
+      },
+      "thinking": {
+        "type": "string"
+      },
+      "tool_calls": {
+        "items": {
+          "properties": {
+            "function": {
+              "properties": {
+                "arguments": {
+                  "additionalProperties": {},
+                  "type": "object",
+                  "x-parser": "gemma4-tool-call"
+                },
+                "name": {
+                  "type": "string"
+                }
+              },
+              "type": "object",
+              "x-regex": "call\\:(?P<name>\\w+)(?P<arguments>\\{.*\\})"
+            },
+            "type": {
+              "const": "function"
+            }
+          },
+          "type": "object"
+        },
+        "type": "array",
+        "x-regex-iterator": "<\\|tool_call>(.*?)<tool_call\\|>"
+      }
+    },
+    "type": "object",
+    "x-regex": "(\\<\\|channel\\>thought\\n(?P<thinking>.*?)\\<channel\\|\\>)?(?P<tool_calls>\\<\\|tool_call\\>.*\\<tool_call\\|\\>)?(?P<content>(?:(?!\\<turn\\|\\>)(?!\\<\\|tool_response\\>).)+)?(?:\\<turn\\|\\>|\\<\\|tool_response\\>)?"
+  },
+  "soc_token": "<|channel>",
+  "sot_token": "<|turn>",
+  "stc_token": "<|tool_call>",
+  "std_token": "<|tool>",
+  "str_token": "<|tool_response>",
+  "think_token": "<|think|>",
+  "tokenizer_class": "GemmaTokenizer",
+  "unk_token": "<unk>"
+}