From 2de01d84a1ffa6d60f5f7294636dc8af10bdc8fd Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Thu, 27 Nov 2025 05:05:28 +0100 Subject: [PATCH 01/27] Diff old read+eval and read-kinds read+eval --- .gitignore | 1 + deps.edn | 2 + src/scicloj/clay/v2/make.clj | 10 +- src/scicloj/clay/v2/notebook.clj | 162 ++++++++++++++++++++++++++----- src/scicloj/clay/v2/read.clj | 22 ++++- src/scicloj/clay/v2/read_old.clj | 143 +++++++++++++++++++++++++++ 6 files changed, 309 insertions(+), 31 deletions(-) create mode 100644 src/scicloj/clay/v2/read_old.clj diff --git a/.gitignore b/.gitignore index b94bdc6f..5610bfcd 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,4 @@ docs/*qmd _quarto.yml book +/read-kinds-diffs diff --git a/deps.edn b/deps.edn index c3834cc3..1a809c73 100644 --- a/deps.edn +++ b/deps.edn @@ -4,6 +4,8 @@ nrepl/nrepl {:mvn/version "1.3.1"} com.cnuernber/charred {:mvn/version "1.037"} read-kinds/read-kinds {:local/root "../read-kinds"} + lambdaisland/deep-diff2 {:mvn/version "2.12.219"} + carocad/parcera {:mvn/version "0.11.6"} org.antlr/antlr4-runtime {:mvn/version "4.7.1"} http-kit/http-kit {:mvn/version "2.8.0"} ring/ring-core {:mvn/version "1.14.1"} diff --git a/src/scicloj/clay/v2/make.clj b/src/scicloj/clay/v2/make.clj index ba11a905..17715f5e 100644 --- a/src/scicloj/clay/v2/make.clj +++ b/src/scicloj/clay/v2/make.clj @@ -17,7 +17,9 @@ [clojure.pprint :as pp] [scicloj.kindly-render.notes.to-html-page :as to-html-page] ;; [hashp.preload] - [scicloj.kindly.v4.api :as kindly])) + [scicloj.kindly.v4.api :as kindly]) + (:import java.time.LocalDateTime + java.time.format.DateTimeFormatter)) (defn spec->source-type [{:keys [source-path]}] (some-> source-path (fs/extension))) @@ -471,8 +473,12 @@ (fs/delete-tree target)) (util.fs/copy-tree-no-clj subdir target))))))) +(defn ts [] + (.format (LocalDateTime/now) + (DateTimeFormatter/ofPattern "yyyy-MM-dd-HH-mm-ss~N"))) + (defn make! [spec] - (let [config (config/config spec) + (let [config (config/config (assoc spec :diff/timestamp (ts))) {:keys [single-form single-value]} spec {:keys [main-spec single-ns-specs]} (extract-specs config spec) {:keys [ide browse show book base-target-path clean-up-target-dir live-reload]} main-spec diff --git a/src/scicloj/clay/v2/notebook.clj b/src/scicloj/clay/v2/notebook.clj index 25447a88..abba1a5c 100644 --- a/src/scicloj/clay/v2/notebook.clj +++ b/src/scicloj/clay/v2/notebook.clj @@ -6,8 +6,11 @@ [scicloj.clay.v2.item :as item] [scicloj.clay.v2.prepare :as prepare] [scicloj.clay.v2.read :as read] + [scicloj.clay.v2.read-old :as read-old] + [lambdaisland.deep-diff2 :as ddiff] [scicloj.kindly.v4.api :as kindly] - [scicloj.kindly-advice.v1.api :as kindly-advice]) + [scicloj.kindly-advice.v1.api :as kindly-advice] + [clojure.pprint :as pprint]) (:import (java.io StringWriter))) (set! *warn-on-reflection* true) @@ -98,8 +101,6 @@ "Captures stdout and stderr while evaluating a note" [{:as note :keys [code form]}] - note - #_ (let [out (StringWriter.) err (StringWriter.) note (try @@ -138,9 +139,10 @@ (defn complete [{:as note :keys [comment?]}] (let [completed (cond-> note - (not (or comment? (contains? note :value))) + (not (or comment? (contains? note :value) + (contains? note ::read/read-kinds))) (read-eval-capture))] - (cond-> completed + (cond-> (dissoc completed ::read/read-kinds) (and (not comment?) (contains? completed :value)) (kindly-advice/advise)))) @@ -406,25 +408,24 @@ :full-target-path :qmd-target-path :kindly/options - :format])] + :format + ::read/read-kinds])] (doall (for [note notes] (complete (kindly/deep-merge opts note)))))) (defn relevant-notes [{:keys [full-source-path - single-form - single-value - smart-sync - pprint-margin] - :or {pprint-margin pp/*print-right-margin*}}] - (let [{:keys [code first-line-of-change]} (some-> full-source-path slurp-and-compare) - notes (->> (cond single-value (conj (when code - [{:form (read/read-ns-form code)}]) - {:value single-value}) - single-form (conj (when code - [{:form (read/read-ns-form code)}]) - {:form single-form}) - :else (read/->notes code)) + single-form + single-value + smart-sync + pprint-margin] + :or {pprint-margin pp/*print-right-margin*} + :as spec}] + (let [{:keys [code first-line-of-change]} (some-> full-source-path + slurp-and-compare) + notes (->> (if (contains? spec ::read/read-kinds) + (read/->notes (assoc spec :code code)) + (read-old/->notes (assoc spec :code code))) (map-indexed (fn [i {:as note :keys [code]}] (merge note @@ -464,6 +465,71 @@ "seconds") result#)) +(defn diff-read-impls [old new & {:diff/keys [to-files + keep-dirs + to-repl + timestamp + notes] + :keys [full-source-path] + :as spec}] + (assert (or to-files to-repl) "Please pick an output option") + (assert (#{:all :each} notes) "You can diff :all notes or :each individually") + (assert timestamp "Should be assoc'd to spec in scicloj.clay.v2.make/make!") + (let [diff (case notes + :each (mapv #(ddiff/diff %1 %2) old new) + :all (ddiff/diff old new))] + (when (not-empty (ddiff/minimize diff)) + (let [diff-print-fn (case to-files + :deep-diff2/full ddiff/pretty-print + :deep-diff2/minimal (comp ddiff/pretty-print + ddiff/minimize) + ;; No diff, we write old/new for to-files anyways + (:clojure/pprint nil) nil) + repl-print-fn (case to-repl + :deep-diff2/full #(ddiff/pretty-print diff) + :deep-diff2/minimal #(-> diff + ddiff/minimize + ddiff/pretty-print) + :clojure/pprint + #(do (println "old: ") + (pprint/pprint old) + (println "new: ") + (pprint/pprint new)) + nil nil) + dir-fn (fn [] + (let [diffs-base-path (fs/absolutize "read-kinds-diffs") + diffs-path (-> diffs-base-path + (fs/path timestamp)) + diff-base-file (-> full-source-path + (str/replace "/" ".") + (->> (fs/path diffs-path) + str)) + diff-file (str diff-base-file ".edn") + old-file (str diff-base-file ".old.edn") + new-file (str diff-base-file ".new.edn") + diff-dirs (->> (fs/list-dir diffs-base-path + #(fs/directory? % {:nofollow-links true})) + (sort-by fs/last-modified-time))] + (when (number? keep-dirs) + (doseq [dir (take (max 0 (inc (- (count diff-dirs) keep-dirs))) + diff-dirs)] + (fs/delete-tree dir))) + (fs/create-dirs diffs-path) + (println "creating diff file " diff-file " & old/new") + (when diff-print-fn + (spit diff-file + (with-out-str + (println "---------- diff:") + (diff-print-fn diff) + (println "---------- spec:") + (pprint/pprint spec)))) + (spit old-file (with-out-str (pprint/pprint old))) + (spit new-file (with-out-str (pprint/pprint new))))) + diff-out (apply juxt (cond-> [] + to-files (conj dir-fn) + to-repl (conj repl-print-fn)))] + (diff-out))))) + (defn spec-notes [{:as spec :keys [pprint-margin ns-form full-source-path] :or {pprint-margin pp/*print-right-margin*}}] @@ -471,12 +537,58 @@ *warn-on-reflection* *warn-on-reflection* *unchecked-math* *unchecked-math* pp/*print-right-margin* pprint-margin] - (-> (relevant-notes spec) - (complete-notes spec) - (with-out-err-captured) - (log-time (str "Evaluated " - (or (some-> ns-form second name) - (some-> full-source-path fs/file-name))))))) + (let [old (-> (relevant-notes spec) + (complete-notes spec) + (with-out-err-captured) + (log-time (str "Evaluated old read+eval " + (or (some-> ns-form second name) + (some-> full-source-path fs/file-name))))) + read-kinds-spec (assoc spec ::read/read-kinds true) + new (-> (relevant-notes read-kinds-spec) + (complete-notes read-kinds-spec) + (with-out-err-captured) + (log-time (str "Evaluated read-kinds read+eval " + (or (some-> ns-form second name) + (some-> full-source-path fs/file-name)))))] + ;; We can print the plain new and old structures.. + #_(diff-read-impls old new + :diff/to-repl :clojure/pprint + :diff/notes :each + spec) + ;; ..or only differences + (diff-read-impls old new + :diff/to-repl :deep-diff2/minimal + :diff/notes :each + spec) + ;; ..or write old and new files + #_(diff-read-impls old new + :diff/to-files :clojure/pprint + :diff/notes :each + spec) + ;; ..or old, new and full diff files + #_(diff-read-impls old new + :diff/to-files :deep-diff2/full + :diff/notes :each + spec) + ;; ..or old, new and minimal diffs, keeping only the last three runs + #_(diff-read-impls old new + :diff/to-files :deep-diff2/minimal + :diff/keep-dirs 3 + :diff/notes :each + spec) + ;; ..or any combination of the above + #_(diff-read-impls old new + :diff/to-repl :deep-diff2/minimal + :diff/to-files :deep-diff2/full + :diff/keep-dirs 3 + :diff/notes :each + spec) + ;; For few differences diffing all notes can be ok + #_(diff-read-impls old new + :diff/to-repl :deep-diff2/minimal + :diff/notes :all + spec) + new))) (defn items-and-test-forms [notes spec] diff --git a/src/scicloj/clay/v2/read.clj b/src/scicloj/clay/v2/read.clj index 42873097..12a5d1ed 100644 --- a/src/scicloj/clay/v2/read.clj +++ b/src/scicloj/clay/v2/read.clj @@ -1,6 +1,9 @@ (ns scicloj.clay.v2.read (:require [scicloj.read-kinds.notes :as notes] - [scicloj.read-kinds.read :as read])) + [scicloj.read-kinds.read :as read] + [nrepl.core :as nrepl] + [clojure.tools.reader] + [clojure.tools.reader.reader-types])) ;; TODO: not sure if generation is necessary??? @@ -26,9 +29,20 @@ (-> form first (= 'ns))))) first)) -(defn ->notes [code] - (->> (read/read-string-all code) - (into [] notes/notebook-xform))) +;; TODO keep this or something like it +(defn ->notes [{:keys [single-form + single-value + code]}] + (cond single-value (conj (when code + [{:form (read-ns-form code)}]) + {:value single-value}) + ;; TODO Doesn't actually eval the form + single-form (conj (when code + [{:form (read-ns-form code)}]) + {:form single-form}) + :else (->> code + (read/read-string-all) + (into [] notes/notebook-xform)))) ;; TODO: Not needed? read-kinds has a safe-notes wrapper already... (defn ->safe-notes [code] diff --git a/src/scicloj/clay/v2/read_old.clj b/src/scicloj/clay/v2/read_old.clj new file mode 100644 index 00000000..cbc2f596 --- /dev/null +++ b/src/scicloj/clay/v2/read_old.clj @@ -0,0 +1,143 @@ +(ns scicloj.clay.v2.read-old + (:require [clojure.tools.reader] + [clojure.tools.reader.reader-types] + [parcera.core :as parcera] + [clojure.string :as str])) + +(def *generation (atom 0)) + +(defn generation [] + (swap! *generation inc) + @*generation) + +(defn read-forms [code] + (->> code + clojure.tools.reader.reader-types/source-logging-push-back-reader + repeat + (map #(clojure.tools.reader/read % false ::EOF)) + (take-while (partial not= ::EOF)))) + + +(defn read-ns-form [code] + (->> code + read-forms + (filter (fn [form] + (and (sequential? form) + (-> form first (= 'ns))))) + first)) + +(defn read-by-tools-reader [code] + (-> code + ;; avoiding a tools.reader bug -- see: + ;; https://github.com/scicloj/clay/issues/151#issuecomment-2373488031 + (str/replace #"\r\n" "\n") + (->> read-forms + (map (fn [form] + (let [{:keys [line column + end-line end-column + code]} + (meta form)] + (when line ; skip forms with no location info + {:method :tools-reader + :region [line column + end-line end-column] + :code (-> form meta :source) + :form form})))) + (filter some?)))) + +(defn read-by-parcera [code] + (->> code + parcera/ast + rest + (map (fn [node] + (let [node-type (first node) + node-contents (rest node)] + ;; We use parcera only for specific types of + ;; code blocks, that tools.reader does not + ;; provide location info for. + (some->> (when (#{:number :string :symbol :keyword :comment} + node-type) + {:code (first node-contents)}) + (merge {:method :parcera + :region (->> node + meta + ((juxt :parcera.core/start + :parcera.core/end)) + (mapcat (juxt :row + (comp inc + :column))) + vec)} + (when (= :comment node-type) + {:comment? true})))))) + (filter some?))) + +(defn unified-cleaned-comment-block [comment-blocks-sorted-by-region] + {:region (vec (concat (->> comment-blocks-sorted-by-region + first + :region + (take 2)) + (->> comment-blocks-sorted-by-region + last + :region + (drop 2)))) + :code (->> comment-blocks-sorted-by-region + (reduce (fn [{:keys [generated-string max-line]} + {:keys [region code]}] + {:generated-string (str generated-string + (apply str (-> region + first + (- max-line) + (repeat "\n"))) + code) + :max-line (-> region + (nth 2) + (max max-line))}) + {:generated-string "" + :max-line (->> comment-blocks-sorted-by-region + first + :region + first)}) + :generated-string) + :comment? true}) + +(defn ->notes [{:keys [single-form + single-value + code]}] + (cond single-value (conj (when code + [{:form (read-ns-form code)}]) + {:value single-value}) + single-form (conj (when code + [{:form (read-ns-form code)}]) + {:form single-form}) + :else (->> code + ((juxt read-by-tools-reader read-by-parcera)) + (apply concat) + (group-by :region) + (map (fn [[region results]] + (if (-> results count (= 1)) + (first results) + ;; prefer tools.reader over parcera + (->> results + (filter #(-> % :method (= :tools-reader))) + first)))) + (sort-by :region) + (map #(dissoc % :method)) + (partition-by :comment?) + (mapcat (fn [part] + (if (-> part first :comment?) + [(unified-cleaned-comment-block part)] + part))) + (mapv (let [g (generation)] + (fn [note-data] + (-> note-data + (assoc :gen g)))))))) + + +(defn ->safe-notes [code] + (try + (->notes code) + (catch Exception e + (println :invalid-notes (-> e + Throwable->map + (select-keys [:cause :data]))) + nil))) From b3d462bb598339302da484a5fcec87def1b04d09 Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Thu, 27 Nov 2025 05:31:08 +0100 Subject: [PATCH 02/27] Add one difference example --- src/scicloj/clay/v2/notebook.clj | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/scicloj/clay/v2/notebook.clj b/src/scicloj/clay/v2/notebook.clj index abba1a5c..4a2eb5ad 100644 --- a/src/scicloj/clay/v2/notebook.clj +++ b/src/scicloj/clay/v2/notebook.clj @@ -557,6 +557,11 @@ spec) ;; ..or only differences (diff-read-impls old new + :diff/to-repl :deep-diff2/minimal + :diff/notes :each + spec) + ;; ..or only one difference + #_(diff-read-impls (take 1 old) (take 1 new) :diff/to-repl :deep-diff2/minimal :diff/notes :each spec) From bfc0f8b0b8a6db26608348dc3b4a175e906c06e6 Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Fri, 28 Nov 2025 01:40:04 +0100 Subject: [PATCH 03/27] Moved to scicloj.clay.v2.util.diff, refactored, fixed --- src/scicloj/clay/v2/notebook.clj | 130 ++++++------------------------ src/scicloj/clay/v2/util/diff.clj | 79 ++++++++++++++++++ 2 files changed, 105 insertions(+), 104 deletions(-) create mode 100644 src/scicloj/clay/v2/util/diff.clj diff --git a/src/scicloj/clay/v2/notebook.clj b/src/scicloj/clay/v2/notebook.clj index 4a2eb5ad..1e7f46db 100644 --- a/src/scicloj/clay/v2/notebook.clj +++ b/src/scicloj/clay/v2/notebook.clj @@ -7,10 +7,9 @@ [scicloj.clay.v2.prepare :as prepare] [scicloj.clay.v2.read :as read] [scicloj.clay.v2.read-old :as read-old] - [lambdaisland.deep-diff2 :as ddiff] [scicloj.kindly.v4.api :as kindly] [scicloj.kindly-advice.v1.api :as kindly-advice] - [clojure.pprint :as pprint]) + [scicloj.clay.v2.util.diff :as diff]) (:import (java.io StringWriter))) (set! *warn-on-reflection* true) @@ -465,71 +464,6 @@ "seconds") result#)) -(defn diff-read-impls [old new & {:diff/keys [to-files - keep-dirs - to-repl - timestamp - notes] - :keys [full-source-path] - :as spec}] - (assert (or to-files to-repl) "Please pick an output option") - (assert (#{:all :each} notes) "You can diff :all notes or :each individually") - (assert timestamp "Should be assoc'd to spec in scicloj.clay.v2.make/make!") - (let [diff (case notes - :each (mapv #(ddiff/diff %1 %2) old new) - :all (ddiff/diff old new))] - (when (not-empty (ddiff/minimize diff)) - (let [diff-print-fn (case to-files - :deep-diff2/full ddiff/pretty-print - :deep-diff2/minimal (comp ddiff/pretty-print - ddiff/minimize) - ;; No diff, we write old/new for to-files anyways - (:clojure/pprint nil) nil) - repl-print-fn (case to-repl - :deep-diff2/full #(ddiff/pretty-print diff) - :deep-diff2/minimal #(-> diff - ddiff/minimize - ddiff/pretty-print) - :clojure/pprint - #(do (println "old: ") - (pprint/pprint old) - (println "new: ") - (pprint/pprint new)) - nil nil) - dir-fn (fn [] - (let [diffs-base-path (fs/absolutize "read-kinds-diffs") - diffs-path (-> diffs-base-path - (fs/path timestamp)) - diff-base-file (-> full-source-path - (str/replace "/" ".") - (->> (fs/path diffs-path) - str)) - diff-file (str diff-base-file ".edn") - old-file (str diff-base-file ".old.edn") - new-file (str diff-base-file ".new.edn") - diff-dirs (->> (fs/list-dir diffs-base-path - #(fs/directory? % {:nofollow-links true})) - (sort-by fs/last-modified-time))] - (when (number? keep-dirs) - (doseq [dir (take (max 0 (inc (- (count diff-dirs) keep-dirs))) - diff-dirs)] - (fs/delete-tree dir))) - (fs/create-dirs diffs-path) - (println "creating diff file " diff-file " & old/new") - (when diff-print-fn - (spit diff-file - (with-out-str - (println "---------- diff:") - (diff-print-fn diff) - (println "---------- spec:") - (pprint/pprint spec)))) - (spit old-file (with-out-str (pprint/pprint old))) - (spit new-file (with-out-str (pprint/pprint new))))) - diff-out (apply juxt (cond-> [] - to-files (conj dir-fn) - to-repl (conj repl-print-fn)))] - (diff-out))))) - (defn spec-notes [{:as spec :keys [pprint-margin ns-form full-source-path] :or {pprint-margin pp/*print-right-margin*}}] @@ -550,49 +484,37 @@ (log-time (str "Evaluated read-kinds read+eval " (or (some-> ns-form second name) (some-> full-source-path fs/file-name)))))] - ;; We can print the plain new and old structures.. - #_(diff-read-impls old new - :diff/to-repl :clojure/pprint - :diff/notes :each - spec) + ;; We can print the plain new and old notes.. + #_(diff/notes old new + :diff/to-repl :clojure/pprint + spec) ;; ..or only differences - (diff-read-impls old new - :diff/to-repl :deep-diff2/minimal - :diff/notes :each - spec) + (diff/notes old new + :diff/to-repl :deep-diff2/minimal + spec) ;; ..or only one difference - #_(diff-read-impls (take 1 old) (take 1 new) - :diff/to-repl :deep-diff2/minimal - :diff/notes :each - spec) + #_(diff/notes (take 1 old) (take 1 new) + :diff/to-repl :deep-diff2/minimal + spec) ;; ..or write old and new files - #_(diff-read-impls old new - :diff/to-files :clojure/pprint - :diff/notes :each - spec) + #_(diff/notes old new + :diff/to-files :clojure/pprint + spec) ;; ..or old, new and full diff files - #_(diff-read-impls old new - :diff/to-files :deep-diff2/full - :diff/notes :each - spec) + #_(diff/notes old new + :diff/to-files :deep-diff2/full + spec) ;; ..or old, new and minimal diffs, keeping only the last three runs - #_(diff-read-impls old new - :diff/to-files :deep-diff2/minimal - :diff/keep-dirs 3 - :diff/notes :each - spec) + #_(diff/notes old new + :diff/to-files :deep-diff2/minimal + :diff/keep-dirs 3 + spec) ;; ..or any combination of the above - #_(diff-read-impls old new - :diff/to-repl :deep-diff2/minimal - :diff/to-files :deep-diff2/full - :diff/keep-dirs 3 - :diff/notes :each - spec) - ;; For few differences diffing all notes can be ok - #_(diff-read-impls old new - :diff/to-repl :deep-diff2/minimal - :diff/notes :all - spec) + #_(diff/notes old new + :diff/to-repl :deep-diff2/minimal + :diff/to-files :deep-diff2/full + :diff/keep-dirs 3 + spec) new))) (defn items-and-test-forms diff --git a/src/scicloj/clay/v2/util/diff.clj b/src/scicloj/clay/v2/util/diff.clj new file mode 100644 index 00000000..ee5c570c --- /dev/null +++ b/src/scicloj/clay/v2/util/diff.clj @@ -0,0 +1,79 @@ +(ns scicloj.clay.v2.util.diff + (:require [clojure.string :as str] + [clojure.pprint :as pp] + [babashka.fs :as fs] + [lambdaisland.deep-diff2 :as ddiff])) + +(defn- print-diffs [diff-print-fn note-diffs] + (doseq [diff note-diffs] + (diff-print-fn diff))) + +(defn- write-diff-files [old new note-diffs diff-print-fn print-fn + {:diff/keys [keep-dirs + timestamp] + :keys [full-source-path] + :as spec}] + (let [diffs-base-path (fs/absolutize "read-kinds-diffs") + diffs-path (-> diffs-base-path + (fs/path timestamp)) + diff-base-file (-> full-source-path + (str/replace "/" ".") + (->> (fs/path diffs-path) + str)) + diff-file (str diff-base-file ".edn") + old-file (str diff-base-file ".old.edn") + new-file (str diff-base-file ".new.edn") + diff-dirs (->> (fs/list-dir diffs-base-path + #(fs/directory? % {:nofollow-links true})) + (sort-by fs/last-modified-time))] + (when (number? keep-dirs) + (doseq [dir (take (max 0 (inc (- (count diff-dirs) keep-dirs))) + diff-dirs)] + (when (= (fs/parent dir) diffs-base-path) + (fs/delete-tree dir)))) + (fs/create-dirs diffs-path) + (println "creating diff file " diff-file " & old/new") + (when diff-print-fn + (spit diff-file + (with-out-str + (print-diffs diff-print-fn note-diffs)))) + (spit old-file (with-out-str (print-fn old))) + (spit new-file (with-out-str (print-fn new))))) + +(defn- pad-notes [old new] + (let [pad-to #(take (max 0 (- (count %1) (count %2))) (repeat {}))] + [(into [] (concat old (pad-to new old))) + (into [] (concat new (pad-to old new)))])) + +(defn notes [old new & {:diff/keys [to-files + to-repl + timestamp] + :as spec}] + (assert (or to-files to-repl) "Please pick an output option") + (assert timestamp "Should be assoc'd to spec in scicloj.clay.v2.make/make!") + (let [[old new] (pad-notes old new) + note-diffs (mapv ddiff/diff old new)] + (when (not-empty (ddiff/minimize note-diffs)) + (let [file-diff-print-fn (case to-files + :deep-diff2/full ddiff/pretty-print + :deep-diff2/minimal (comp ddiff/pretty-print + ddiff/minimize) + ;; No diff, we write old/new for to-files anyways + (:clojure/pprint nil) nil) + print-fn pp/pprint + repl-print-fn (case to-repl + :deep-diff2/full #(print-diffs ddiff/pretty-print + note-diffs) + :deep-diff2/minimal #(print-diffs (comp ddiff/pretty-print + ddiff/minimize) + note-diffs) + :clojure/pprint #(doseq [[old' new'] (map vector old new)] + (println "--------- old: ") + (print-fn old') + (println "--------- new: ") + (print-fn new')) + nil nil)] + (when to-files + (write-diff-files old new note-diffs file-diff-print-fn print-fn spec)) + (when to-repl + (repl-print-fn)))))) From 4c7092be96c5f4dde0f669e4a955e299e7641435 Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Fri, 28 Nov 2025 02:05:55 +0100 Subject: [PATCH 04/27] Add pre read-kinds notebook.clj as notebook_old.clj --- src/scicloj/clay/v2/notebook_old.clj | 490 +++++++++++++++++++++++++++ 1 file changed, 490 insertions(+) create mode 100644 src/scicloj/clay/v2/notebook_old.clj diff --git a/src/scicloj/clay/v2/notebook_old.clj b/src/scicloj/clay/v2/notebook_old.clj new file mode 100644 index 00000000..2010b644 --- /dev/null +++ b/src/scicloj/clay/v2/notebook_old.clj @@ -0,0 +1,490 @@ +(ns scicloj.clay.v2.notebook-old + (:require [clojure.string :as str] + [clojure.pprint :as pp] + [babashka.fs :as fs] + [scicloj.clay.v2.util.path :as path] + [scicloj.clay.v2.item :as item] + [scicloj.clay.v2.prepare :as prepare] + [scicloj.clay.v2.read-old :as read] + [scicloj.kindly.v4.api :as kindly] + [scicloj.kindly-advice.v1.api :as kindly-advice]) + (:import (java.io StringWriter))) + +(set! *warn-on-reflection* true) + +(defn deref-if-needed [v] + (if (delay? v) + @v + v)) + +(def hidden-form-starters + #{'ns 'comment + 'def 'defonce 'defn 'defmacro + 'defrecord 'defprotocol 'deftype + 'extend-protocol 'extend + 'require}) + +(defn info-line [{:keys [full-source-path + remote-repo]}] + (let [relative-file-path (path/path-relative-to-repo full-source-path)] + (item/info-line {:path relative-file-path + :url (some-> remote-repo (path/file-git-url relative-file-path))}))) + +(defn narrowed? [code] + (some-> code + (str/includes? ",,"))) + +(defn narrower? [code] + (some-> code + (str/includes? ",,,"))) + +(defn ns-form? [form] + (and (sequential? form) + (-> form first (= 'ns)))) + +(defn str-and-reset! [w] + (when (instance? StringWriter *out*) + (locking w + (let [s (str w)] + (.setLength (.getBuffer ^StringWriter w) 0) + s)))) + +(def ^:dynamic *out-orig* *out*) + +(defn maybe-println-orig [s] + (when (seq s) + (binding [*out* *out-orig*] + (print s) + (flush)))) + +(def ^:dynamic *err-orig* *err*) + +(defn maybe-err-orig [s] + (when (seq s) + (binding [*out* *err-orig*] + (print s) + (flush)))) + +(defn read-eval-capture + "Captures stdout and stderr while evaluating a note" + [{:as note + :keys [code form]}] + (let [out (StringWriter.) + err (StringWriter.) + note (try + (let [x (binding [*out* out + *err* err] + (cond form (-> form + eval + deref-if-needed) + code (-> code + read-string + eval + deref-if-needed)))] + (assoc note :value x)) + (catch Throwable ex + (assoc note :exception ex))) + out-str (str out) + err-str (str err) + ;; A notebook may have also printed from a thread, + ;; *out* and *err* are replaced with StringWriters in with-out-err-capture + global-out (str-and-reset! *out*) + global-err (str-and-reset! *err*) + ;; Don't show output from requiring other namespaces + show (not (ns-form? form))] + (maybe-println-orig out-str) + (maybe-err-orig err-str) + (maybe-println-orig global-out) + (maybe-err-orig global-err) + (if show + (cond-> note + (seq out-str) (assoc :out out-str) + (seq err-str) (assoc :err err-str) + (seq global-out) (assoc :global-out global-out) + (seq global-err) (assoc :global-err global-err)) + note))) + +(defn complete [{:as note + :keys [comment?]}] + (let [completed (cond-> note + (not (or comment? (contains? note :value))) + (read-eval-capture))] + (cond-> completed + (and (not comment?) (contains? completed :value)) + (kindly-advice/advise)))) + +(defn comment->item [comment] + (-> comment + (str/split #"\n") + (->> (map #(-> % + (str/replace + #"^;+\s?" "") + (str/replace + #"^#" "\n#"))) + (str/join "\n")) + item/md)) + +(defn hide-code? [{:as note :keys [code form value kind narrowed]} {:as opts :keys [hide-code]}] + (or hide-code + narrowed + (-> form meta :kindly/hide-code) + (-> form meta :kindly/hide-code?) ; legacy convention + (-> value meta :kindly/hide-code) + (-> value meta :kindly/hide-code?) ; legacy convention + (when kind + (some-> note + :kindly/options + :kinds-that-hide-code + kind)) + (nil? code))) + +(defn hide-value? [{:as complete-note :keys [form value kind]} + {:as opts :keys [hide-nils hide-vars]}] + (or (and (sequential? form) + (-> form first hidden-form-starters)) + (= kind :kind/hidden) + (and hide-nils (nil? value)) + (and hide-vars (var? value)))) + +(defn side-by-side-items [{:as spec :keys [format]} code-item value-items] + ;; markdown grids are not structurally nested, but hiccup grids are + (if (= :quarto (first format)) + `[{:md "::: {.grid .clay-side-by-side}"} + {:md "::: {.g-col-6}"} + ~code-item + {:md ":::"} + {:md "::: {.g-col-6}"} + ~@value-items + {:md ":::"} + {:md ":::"}] + [{:hiccup [:div.grid + [:div.g-col-6 (:hiccup code-item)] + (->> (map #(prepare/item->hiccup % spec) value-items) + (into [:div.g-col-6]))] + :deps (set (mapcat :deps value-items))}])) + +(defn note-to-items [{:as note + :keys [comment? + code + exception + err + out + global-err + global-out + kindly/options]} + {:as opts}] + (if (and comment? code) + [(comment->item code)] + (let [code-item (when-not (hide-code? note opts) + (item/source-clojure code)) + {:keys [exception-continue]} opts + value-items (cond-> [] + err (conj (item/print-output "ERR" err)) + out (conj (item/print-output "OUT" out)) + global-err (conj (item/print-output "THREAD ERR" global-err)) + global-out (conj (item/print-output "THREAD OUT" global-out)) + exception (conj (item/print-throwable exception exception-continue)) + (and (contains? note :value) + (not (hide-value? note opts))) + (into (-> note + (update :value deref-if-needed) + (prepare/prepare-or-pprint))))] + (cond (and (not code-item) (empty? value-items)) + [] + + (not code-item) + value-items + + (empty? value-items) + [code-item] + + (= :horizontal (or (:code-and-value opts) + (:code-and-value options))) + (side-by-side-items opts code-item value-items) + + :else + (into [code-item] value-items))))) + +(defn add-info-line [items {:as spec :keys [hide-info-line]}] + (if hide-info-line + items + (let [il (info-line spec)] + (into items [item/separator il])))) + +(defn ->var-name [i line-number] + (symbol (str "v" i + "_l" line-number))) + +(defn ->test-name [i line-number] + (symbol (str "t" i + "_l" line-number))) + +(defn test-last? [complete-note] + (and (-> complete-note + :comment? + not) + (-> complete-note + :kind + (= :kind/test-last)))) + +(defn def-form [var-name form] + (list 'def + var-name + form)) + +(defn clean-test-last-form [form] + (case (str (first form)) + "kind/test-last" (second form) + "kindly/check" (rest form) + ;; else + form)) + +(defn var-based-deftest-form [test-name var-name form] + (let [[f-symbol & args] form] + (list 'deftest + test-name + (concat (list 'is + (concat (list f-symbol + var-name) + args)))))) + +(defn simple-deftest-form [test-name last-form form] + (let [[f-symbol & args] form] + (list 'deftest + test-name + (concat (list 'is + (concat (list f-symbol + last-form) + args)))))) + +(defn test-ns-form [[_ ns-symbol & rest-ns-form]] + (concat (list 'ns + (-> ns-symbol + (str "-generated-test") + symbol)) + (->> rest-ns-form + (map (fn [part] + (if (and (list? part) + (-> part first (= :require))) + (concat part + '[[clojure.test :refer [deftest is]]]) + part)))))) + +(defn first-line-of-change [code new-code] + (if code + (->> [code new-code] + (map str/split-lines) + (apply map =) + (take-while true?) + count) + 0)) + +(def *path->last (atom {})) + +(defn slurp-and-compare [path] + (swap! *path->last + update + path + (fn [{:keys [code]}] + (let [new-code (slurp path)] + {:code new-code + :first-line-of-change (first-line-of-change + code new-code)}))) + (@*path->last path)) + +(defmacro with-out-err-captured + "Evaluates and computes the items for a notebook of notes" + [& body] + ;; For a notebook, we capture output globally, and per note. + ;; see read-eval-capture for why this is relevant. + `(let [out# (StringWriter.) + err# (StringWriter.)] + ;; Threads may inherit only the root binding + (with-redefs [*out* out# + *err* err#] + ;; Futures will inherit the current binding, + ;; which was not affected by altering the root. + (binding [*out* out# + *err* err#] + ~@body)))) + +(defn itemize-notes + "Evaluates and computes the items for a notebook of notes" + [relevant-notes some-narrowed options] + (reduce (fn [{:as aggregation :keys [i + items + test-forms + last-nontest-varname + last-nontest-form]} + complete-note] + (let [{:keys [form region narrowed exception comment?]} complete-note + {:keys [test-mode]} (:kindly/options complete-note) + test-note (test-last? complete-note) + new-items (when (or (not some-narrowed) + narrowed) + (when-not test-note + (note-to-items complete-note options))) + line-number (first region) + varname (->var-name i line-number) + test-form (cond + ;; a deftest form + test-note (vary-meta + (let [test-name (->test-name i line-number) + ctlf (clean-test-last-form form)] + (case test-mode + :sequential (var-based-deftest-form + test-name + last-nontest-varname + ctlf) + :simple (simple-deftest-form + test-name + last-nontest-form + ctlf))) + assoc :test-mode test-mode) + ;; the test ns form + (ns-form? form) (test-ns-form form) + ;; a comment + comment? nil + ;; the regular case, just a def + :else (def-form varname form)) + step {:i (inc i) + :items (into items (remove nil?) new-items) + :test-forms (if test-form + (conj test-forms test-form) + test-forms) + :last-nontest-varname (if (or comment? test-note) + last-nontest-varname + varname) + :last-nontest-form (if (or comment? test-note) + last-nontest-form + form)}] + (if (and exception (not (:exception-continue options))) + (reduced (assoc step :exception exception)) + step))) + ;; initial value + {:i 0 + :items [] + :test-forms [] + :last-nontest-i nil} + ;; sequence + relevant-notes)) + +(defn complete-notes [notes options] + (let [opts (select-keys options + [:base-target-path + :full-target-path + :qmd-target-path + :kindly/options + :format])] + (doall + (for [note notes] + (complete (kindly/deep-merge opts note)))))) + +(defn relevant-notes [{:keys [full-source-path + single-form + single-value + smart-sync + pprint-margin] + :or {pprint-margin pp/*print-right-margin*}}] + (let [{:keys [code first-line-of-change]} (some-> full-source-path slurp-and-compare) + notes (->> (cond single-value (conj (when code + [{:form (read/read-ns-form code)}]) + {:value single-value}) + single-form (conj (when code + [{:form (read/read-ns-form code)}]) + {:form single-form}) + :else (read/->notes code)) + (map-indexed (fn [i {:as note + :keys [code]}] + (merge note + {:i i} + (when-not (:comment? note) + {:narrowed (narrowed? code) + :narrower (narrower? code)}))))) + some-narrowed (some :narrowed notes) + some-narrower (some :narrower notes) + narrowed-indices (when some-narrowed + (->> notes + (map (fn [{:keys [i narrowed]}] + (when narrowed i))) + (remove nil?))) + first-narrowed-index (first narrowed-indices) + last-narrowed-index (last narrowed-indices)] + (cond-> notes + some-narrower + (->> (filter (fn [{:keys [narrower form]}] + (or narrower + (ns-form? form))))) + + (and some-narrowed smart-sync) + (->> (take (inc last-narrowed-index)) + (filter (fn [{:keys [i code form region]}] + (or (ns-form? form) + (>= i first-narrowed-index) + (-> region + (nth 2) ;last region line + (> first-line-of-change))))))))) + +(defmacro log-time [expr msg] + `(let [start# (System/currentTimeMillis) + result# ~expr] + (println "Clay: " ~msg + "in" (/ (- (System/currentTimeMillis) start#) 1000.0) + "seconds") + result#)) + +(defn spec-notes [{:as spec + :keys [pprint-margin ns-form full-source-path] + :or {pprint-margin pp/*print-right-margin*}}] + (binding [*ns* *ns* + *warn-on-reflection* *warn-on-reflection* + *unchecked-math* *unchecked-math* + pp/*print-right-margin* pprint-margin] + (-> (relevant-notes spec) + (complete-notes spec) + (with-out-err-captured) + (log-time (str "Evaluated notebook old " + (or (some-> ns-form second name) + (some-> full-source-path fs/file-name))))))) + +(defn items-and-test-forms + [notes spec] + (let [some-narrowed (some :narrowed notes)] + (-> notes + (itemize-notes some-narrowed spec) + (update :items add-info-line spec) + (update :test-forms + ;; Leave the test-form only when + ;; at least one of them is a `deftest`. + (if some-narrowed + (constantly nil) + (fn [test-forms] + (let [deftest-forms (->> test-forms + (filter #(-> % first (= 'deftest))))] + (when ;; there are some actual test forms + (seq deftest-forms) + #_(prn [:test-forms test-forms + :deftest-forms deftest-forms + :check (->> deftest-forms + (map (comp :test-mode meta)) + (some #(= % :sequential)))]) + (if (->> deftest-forms + (map (comp :test-mode meta)) + (some #(= % :sequential))) + ;; Some tests are of `:sequential` mode, + ;; so we need all the intermediate `def` forms, + ;; not just the `deftest` forms. + test-forms + ;; Else - all tests are of `:simple` mode, + ;; so we only need the `ns` definition and the `deftest` forms. + (cons (first test-forms) + deftest-forms)))))))))) + + +(comment + (-> "notebooks/scratch.clj" + (notebook-items {:full-target-path "docs/scratch.html"})) + + (-> "notebooks/scratch.clj" + (notebook-items {:full-target-path "docs/scratch.html" + :single-form '(+ 1 2)}))) From d09dc8b07d117135364105d30e3c4cdf9c6868c6 Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Fri, 28 Nov 2025 02:13:24 +0100 Subject: [PATCH 05/27] Revert read_old.clj --- src/scicloj/clay/v2/read_old.clj | 54 ++++++++++++++------------------ 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/src/scicloj/clay/v2/read_old.clj b/src/scicloj/clay/v2/read_old.clj index cbc2f596..6fd6bcce 100644 --- a/src/scicloj/clay/v2/read_old.clj +++ b/src/scicloj/clay/v2/read_old.clj @@ -100,37 +100,29 @@ :generated-string) :comment? true}) -(defn ->notes [{:keys [single-form - single-value - code]}] - (cond single-value (conj (when code - [{:form (read-ns-form code)}]) - {:value single-value}) - single-form (conj (when code - [{:form (read-ns-form code)}]) - {:form single-form}) - :else (->> code - ((juxt read-by-tools-reader read-by-parcera)) - (apply concat) - (group-by :region) - (map (fn [[region results]] - (if (-> results count (= 1)) - (first results) - ;; prefer tools.reader over parcera - (->> results - (filter #(-> % :method (= :tools-reader))) - first)))) - (sort-by :region) - (map #(dissoc % :method)) - (partition-by :comment?) - (mapcat (fn [part] - (if (-> part first :comment?) - [(unified-cleaned-comment-block part)] - part))) - (mapv (let [g (generation)] - (fn [note-data] - (-> note-data - (assoc :gen g)))))))) +(defn ->notes [code] + (->> code + ((juxt read-by-tools-reader read-by-parcera)) + (apply concat) + (group-by :region) + (map (fn [[region results]] + (if (-> results count (= 1)) + (first results) + ;; prefer tools.reader over parcera + (->> results + (filter #(-> % :method (= :tools-reader))) + first)))) + (sort-by :region) + (map #(dissoc % :method)) + (partition-by :comment?) + (mapcat (fn [part] + (if (-> part first :comment?) + [(unified-cleaned-comment-block part)] + part))) + (mapv (let [g (generation)] + (fn [note-data] + (-> note-data + (assoc :gen g))))))) (defn ->safe-notes [code] From 44f7813f85a7bbbc3a6c1b1966c504fa36fcb097 Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Fri, 28 Nov 2025 02:22:51 +0100 Subject: [PATCH 06/27] Diff on old and new notebooks --- src/scicloj/clay/v2/notebook.clj | 29 ++++++++++------------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/src/scicloj/clay/v2/notebook.clj b/src/scicloj/clay/v2/notebook.clj index 1e7f46db..adf3b193 100644 --- a/src/scicloj/clay/v2/notebook.clj +++ b/src/scicloj/clay/v2/notebook.clj @@ -5,8 +5,8 @@ [scicloj.clay.v2.util.path :as path] [scicloj.clay.v2.item :as item] [scicloj.clay.v2.prepare :as prepare] + [scicloj.clay.v2.notebook-old :as notebook-old] [scicloj.clay.v2.read :as read] - [scicloj.clay.v2.read-old :as read-old] [scicloj.kindly.v4.api :as kindly] [scicloj.kindly-advice.v1.api :as kindly-advice] [scicloj.clay.v2.util.diff :as diff]) @@ -100,7 +100,8 @@ "Captures stdout and stderr while evaluating a note" [{:as note :keys [code form]}] - (let [out (StringWriter.) + note + #_(let [out (StringWriter.) err (StringWriter.) note (try (let [x (binding [*out* out @@ -138,10 +139,9 @@ (defn complete [{:as note :keys [comment?]}] (let [completed (cond-> note - (not (or comment? (contains? note :value) - (contains? note ::read/read-kinds))) + (not (or comment? (contains? note :value))) (read-eval-capture))] - (cond-> (dissoc completed ::read/read-kinds) + (cond-> completed (and (not comment?) (contains? completed :value)) (kindly-advice/advise)))) @@ -407,8 +407,7 @@ :full-target-path :qmd-target-path :kindly/options - :format - ::read/read-kinds])] + :format])] (doall (for [note notes] (complete (kindly/deep-merge opts note)))))) @@ -422,9 +421,7 @@ :as spec}] (let [{:keys [code first-line-of-change]} (some-> full-source-path slurp-and-compare) - notes (->> (if (contains? spec ::read/read-kinds) - (read/->notes (assoc spec :code code)) - (read-old/->notes (assoc spec :code code))) + notes (->> (read/->notes (assoc spec :code code)) (map-indexed (fn [i {:as note :keys [code]}] (merge note @@ -471,17 +468,11 @@ *warn-on-reflection* *warn-on-reflection* *unchecked-math* *unchecked-math* pp/*print-right-margin* pprint-margin] - (let [old (-> (relevant-notes spec) + (let [old (notebook-old/spec-notes spec) + new (-> (relevant-notes spec) (complete-notes spec) (with-out-err-captured) - (log-time (str "Evaluated old read+eval " - (or (some-> ns-form second name) - (some-> full-source-path fs/file-name))))) - read-kinds-spec (assoc spec ::read/read-kinds true) - new (-> (relevant-notes read-kinds-spec) - (complete-notes read-kinds-spec) - (with-out-err-captured) - (log-time (str "Evaluated read-kinds read+eval " + (log-time (str "Evaluated notebook with read-kinds " (or (some-> ns-form second name) (some-> full-source-path fs/file-name)))))] ;; We can print the plain new and old notes.. From c322416693146c3e897ea45db2c1cebf78ee0364 Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Fri, 28 Nov 2025 02:23:20 +0100 Subject: [PATCH 07/27] Print with Clay: --- src/scicloj/clay/v2/util/diff.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scicloj/clay/v2/util/diff.clj b/src/scicloj/clay/v2/util/diff.clj index ee5c570c..dabfe82e 100644 --- a/src/scicloj/clay/v2/util/diff.clj +++ b/src/scicloj/clay/v2/util/diff.clj @@ -32,7 +32,7 @@ (when (= (fs/parent dir) diffs-base-path) (fs/delete-tree dir)))) (fs/create-dirs diffs-path) - (println "creating diff file " diff-file " & old/new") + (println "Clay: Creating diff file " diff-file " & old/new") (when diff-print-fn (spit diff-file (with-out-str @@ -76,4 +76,5 @@ (when to-files (write-diff-files old new note-diffs file-diff-print-fn print-fn spec)) (when to-repl + (println "Clay: Notes diff") (repl-print-fn)))))) From b3428a3f77af532c4180afcd052e123f0665597c Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Fri, 28 Nov 2025 02:55:15 +0100 Subject: [PATCH 08/27] Take out some keys unique to new/old notes --- src/scicloj/clay/v2/notebook.clj | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/scicloj/clay/v2/notebook.clj b/src/scicloj/clay/v2/notebook.clj index adf3b193..2f7f0f24 100644 --- a/src/scicloj/clay/v2/notebook.clj +++ b/src/scicloj/clay/v2/notebook.clj @@ -461,6 +461,18 @@ "seconds") result#)) +(defn notes-dissoc-old [notes] + (->> notes + (into (empty notes) + (map #(-> % + (dissoc :gen)))))) + +(defn notes-dissoc-new [notes] + (->> notes + (into (empty notes) + (map #(-> % + (dissoc :line :column)))))) + (defn spec-notes [{:as spec :keys [pprint-margin ns-form full-source-path] :or {pprint-margin pp/*print-right-margin*}}] @@ -468,13 +480,15 @@ *warn-on-reflection* *warn-on-reflection* *unchecked-math* *unchecked-math* pp/*print-right-margin* pprint-margin] - (let [old (notebook-old/spec-notes spec) + (let [old (-> (notebook-old/spec-notes spec) + (notes-dissoc-old)) new (-> (relevant-notes spec) (complete-notes spec) (with-out-err-captured) (log-time (str "Evaluated notebook with read-kinds " (or (some-> ns-form second name) - (some-> full-source-path fs/file-name)))))] + (some-> full-source-path fs/file-name)))) + (notes-dissoc-new))] ;; We can print the plain new and old notes.. #_(diff/notes old new :diff/to-repl :clojure/pprint From 442c9709b6a44527c518ad105ac05fec650ea71d Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Fri, 28 Nov 2025 03:31:51 +0100 Subject: [PATCH 09/27] Tricked by seqs --- src/scicloj/clay/v2/notebook.clj | 4 ++-- src/scicloj/clay/v2/read.clj | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/scicloj/clay/v2/notebook.clj b/src/scicloj/clay/v2/notebook.clj index 2f7f0f24..c35e83e6 100644 --- a/src/scicloj/clay/v2/notebook.clj +++ b/src/scicloj/clay/v2/notebook.clj @@ -463,13 +463,13 @@ (defn notes-dissoc-old [notes] (->> notes - (into (empty notes) + (into [] (map #(-> % (dissoc :gen)))))) (defn notes-dissoc-new [notes] (->> notes - (into (empty notes) + (into [] (map #(-> % (dissoc :line :column)))))) diff --git a/src/scicloj/clay/v2/read.clj b/src/scicloj/clay/v2/read.clj index 12a5d1ed..dfa576ce 100644 --- a/src/scicloj/clay/v2/read.clj +++ b/src/scicloj/clay/v2/read.clj @@ -1,7 +1,6 @@ (ns scicloj.clay.v2.read (:require [scicloj.read-kinds.notes :as notes] [scicloj.read-kinds.read :as read] - [nrepl.core :as nrepl] [clojure.tools.reader] [clojure.tools.reader.reader-types])) From a3dfe9e0bb395a10d100add440e463ad16a81e9d Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Fri, 28 Nov 2025 03:32:01 +0100 Subject: [PATCH 10/27] Copy current diff to diffs-folder toplevel --- src/scicloj/clay/v2/util/diff.clj | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/scicloj/clay/v2/util/diff.clj b/src/scicloj/clay/v2/util/diff.clj index dabfe82e..885351cc 100644 --- a/src/scicloj/clay/v2/util/diff.clj +++ b/src/scicloj/clay/v2/util/diff.clj @@ -9,10 +9,10 @@ (diff-print-fn diff))) (defn- write-diff-files [old new note-diffs diff-print-fn print-fn - {:diff/keys [keep-dirs - timestamp] - :keys [full-source-path] - :as spec}] + {:diff/keys [keep-dirs + timestamp] + :keys [full-source-path] + :as spec}] (let [diffs-base-path (fs/absolutize "read-kinds-diffs") diffs-path (-> diffs-base-path (fs/path timestamp)) @@ -38,7 +38,13 @@ (with-out-str (print-diffs diff-print-fn note-diffs)))) (spit old-file (with-out-str (print-fn old))) - (spit new-file (with-out-str (print-fn new))))) + (spit new-file (with-out-str (print-fn new))) + (doseq [file (fs/list-dir diffs-base-path + #(fs/regular-file? % {:nofollow-links true}))] + (fs/delete file)) + (doseq [file (fs/list-dir diffs-path + #(fs/regular-file? % {:nofollow-links true}))] + (fs/copy file diffs-base-path)))) (defn- pad-notes [old new] (let [pad-to #(take (max 0 (- (count %1) (count %2))) (repeat {}))] @@ -46,9 +52,9 @@ (into [] (concat new (pad-to old new)))])) (defn notes [old new & {:diff/keys [to-files - to-repl - timestamp] - :as spec}] + to-repl + timestamp] + :as spec}] (assert (or to-files to-repl) "Please pick an output option") (assert timestamp "Should be assoc'd to spec in scicloj.clay.v2.make/make!") (let [[old new] (pad-notes old new) From ea1003d8994398aed1ffa32905d33c0a0c87443f Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Fri, 28 Nov 2025 04:13:15 +0100 Subject: [PATCH 11/27] Print convenience --- src/scicloj/clay/v2/util/diff.clj | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/scicloj/clay/v2/util/diff.clj b/src/scicloj/clay/v2/util/diff.clj index 885351cc..46f579d6 100644 --- a/src/scicloj/clay/v2/util/diff.clj +++ b/src/scicloj/clay/v2/util/diff.clj @@ -32,7 +32,7 @@ (when (= (fs/parent dir) diffs-base-path) (fs/delete-tree dir)))) (fs/create-dirs diffs-path) - (println "Clay: Creating diff file " diff-file " & old/new") + (println "Clay: Creating diff file" diff-file "& old/new") (when diff-print-fn (spit diff-file (with-out-str @@ -79,8 +79,8 @@ (println "--------- new: ") (print-fn new')) nil nil)] - (when to-files - (write-diff-files old new note-diffs file-diff-print-fn print-fn spec)) (when to-repl - (println "Clay: Notes diff") - (repl-print-fn)))))) + (println "Clay: Notes diff") + (repl-print-fn)) + (when to-files + (write-diff-files old new note-diffs file-diff-print-fn print-fn spec)))))) From b83eaadab06c8e8c1d7b793b9d162580d0b080eb Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Fri, 28 Nov 2025 04:14:07 +0100 Subject: [PATCH 12/27] Include notebook path in diff folder name --- src/scicloj/clay/v2/util/diff.clj | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/scicloj/clay/v2/util/diff.clj b/src/scicloj/clay/v2/util/diff.clj index 46f579d6..ba578620 100644 --- a/src/scicloj/clay/v2/util/diff.clj +++ b/src/scicloj/clay/v2/util/diff.clj @@ -14,13 +14,13 @@ :keys [full-source-path] :as spec}] (let [diffs-base-path (fs/absolutize "read-kinds-diffs") + source-name (-> full-source-path (str/replace "/" ".")) diffs-path (-> diffs-base-path - (fs/path timestamp)) - diff-base-file (-> full-source-path - (str/replace "/" ".") - (->> (fs/path diffs-path) - str)) - diff-file (str diff-base-file ".edn") + (fs/path (str source-name "~" timestamp))) + diff-base-file (->> source-name + (fs/path diffs-path) + str) + diff-file (str diff-base-file ".diff.edn") old-file (str diff-base-file ".old.edn") new-file (str diff-base-file ".new.edn") diff-dirs (->> (fs/list-dir diffs-base-path From a1710f3c76266dd3968ee7467f687d1157726684 Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Fri, 28 Nov 2025 08:03:55 +0100 Subject: [PATCH 13/27] complete not needed for read-kinds --- src/scicloj/clay/v2/notebook.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scicloj/clay/v2/notebook.clj b/src/scicloj/clay/v2/notebook.clj index c35e83e6..db398f6c 100644 --- a/src/scicloj/clay/v2/notebook.clj +++ b/src/scicloj/clay/v2/notebook.clj @@ -410,7 +410,7 @@ :format])] (doall (for [note notes] - (complete (kindly/deep-merge opts note)))))) + (kindly/deep-merge opts note))))) (defn relevant-notes [{:keys [full-source-path single-form From 8c481f0ab255c2167875b46b13cd9145ff17bba7 Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Fri, 28 Nov 2025 08:05:27 +0100 Subject: [PATCH 14/27] out and err captured by read-kinds --- src/scicloj/clay/v2/notebook.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scicloj/clay/v2/notebook.clj b/src/scicloj/clay/v2/notebook.clj index db398f6c..6f6be87d 100644 --- a/src/scicloj/clay/v2/notebook.clj +++ b/src/scicloj/clay/v2/notebook.clj @@ -484,7 +484,8 @@ (notes-dissoc-old)) new (-> (relevant-notes spec) (complete-notes spec) - (with-out-err-captured) + ;; TODO Read kinds should be doing this + #_(with-out-err-captured) (log-time (str "Evaluated notebook with read-kinds " (or (some-> ns-form second name) (some-> full-source-path fs/file-name)))) From 5682928ffa42fee7f69b28da22f3f5563221114d Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Fri, 28 Nov 2025 08:06:55 +0100 Subject: [PATCH 15/27] Collapse comments and whitespace notes from read-kinds Maybe we only need this for diffing the notes data --- src/scicloj/clay/v2/read.clj | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/scicloj/clay/v2/read.clj b/src/scicloj/clay/v2/read.clj index dfa576ce..543f6fbe 100644 --- a/src/scicloj/clay/v2/read.clj +++ b/src/scicloj/clay/v2/read.clj @@ -28,6 +28,22 @@ (-> form first (= 'ns))))) first)) +(defn collapse-comments-ws [notes] + (let [collapse (comp #{:kind/whitespace :kind/comment} :kind) + comment? (comp #{:kind/comment} :kind)] + (->> notes + (partition-by (comp boolean collapse)) + (mapcat + #(if (some collapse %) + [(cond-> {:code (str/join (map :code %)) + :kind (if (some comment? %) + :kind/comment + :kind/whitespace)} + (some comment? %) + ;; TODO does :value need :code from whitespace? + (assoc :value (str/join (map :value %))))] + %))))) + ;; TODO keep this or something like it (defn ->notes [{:keys [single-form single-value @@ -41,6 +57,8 @@ {:form single-form}) :else (->> code (read/read-string-all) + ;; TODO maybe optional for diffing + collapse-comments-ws (into [] notes/notebook-xform)))) ;; TODO: Not needed? read-kinds has a safe-notes wrapper already... From 14e0d28f20e48e8c6f39a06bed65add4a8cbffb8 Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Sat, 29 Nov 2025 06:48:35 +0100 Subject: [PATCH 16/27] read missing require --- src/scicloj/clay/v2/read.clj | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/scicloj/clay/v2/read.clj b/src/scicloj/clay/v2/read.clj index 543f6fbe..4e773b4c 100644 --- a/src/scicloj/clay/v2/read.clj +++ b/src/scicloj/clay/v2/read.clj @@ -2,7 +2,8 @@ (:require [scicloj.read-kinds.notes :as notes] [scicloj.read-kinds.read :as read] [clojure.tools.reader] - [clojure.tools.reader.reader-types])) + [clojure.tools.reader.reader-types] + [clojure.string :as str])) ;; TODO: not sure if generation is necessary??? From 5a399ef301496e306724cdb81dcff93dac0676a1 Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Sat, 29 Nov 2025 06:52:02 +0100 Subject: [PATCH 17/27] Improve comment/whitespace collapse a bit --- src/scicloj/clay/v2/read.clj | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/scicloj/clay/v2/read.clj b/src/scicloj/clay/v2/read.clj index 4e773b4c..b3da7677 100644 --- a/src/scicloj/clay/v2/read.clj +++ b/src/scicloj/clay/v2/read.clj @@ -35,15 +35,12 @@ (->> notes (partition-by (comp boolean collapse)) (mapcat - #(if (some collapse %) - [(cond-> {:code (str/join (map :code %)) - :kind (if (some comment? %) - :kind/comment - :kind/whitespace)} - (some comment? %) - ;; TODO does :value need :code from whitespace? - (assoc :value (str/join (map :value %))))] - %))))) + (fn [notes*] + (if (some comment? notes*) + [{:value (str/join (map :value + notes*)) + :kind :kind/comment}] + notes*)))))) ;; TODO keep this or something like it (defn ->notes [{:keys [single-form From ef4f35ab50b829979dc08b129b04de2e4a867533 Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Sat, 29 Nov 2025 06:54:07 +0100 Subject: [PATCH 18/27] Simplified, added no-diff case, informative printing --- src/scicloj/clay/v2/util/diff.clj | 79 ++++++++++++++++++------------- 1 file changed, 47 insertions(+), 32 deletions(-) diff --git a/src/scicloj/clay/v2/util/diff.clj b/src/scicloj/clay/v2/util/diff.clj index ba578620..61c43221 100644 --- a/src/scicloj/clay/v2/util/diff.clj +++ b/src/scicloj/clay/v2/util/diff.clj @@ -6,7 +6,15 @@ (defn- print-diffs [diff-print-fn note-diffs] (doseq [diff note-diffs] - (diff-print-fn diff))) + (when (some-> diff ddiff/minimize not-empty) + (diff-print-fn diff)))) + +(defn- diff-print-fn [k] + (case k + :deep-diff2/full ddiff/pretty-print + :deep-diff2/minimal (comp ddiff/pretty-print + ddiff/minimize) + nil)) (defn- write-diff-files [old new note-diffs diff-print-fn print-fn {:diff/keys [keep-dirs @@ -20,6 +28,7 @@ diff-base-file (->> source-name (fs/path diffs-path) str) + no-diff-file (str diff-base-file ".no-diff") diff-file (str diff-base-file ".diff.edn") old-file (str diff-base-file ".old.edn") new-file (str diff-base-file ".new.edn") @@ -32,13 +41,16 @@ (when (= (fs/parent dir) diffs-base-path) (fs/delete-tree dir)))) (fs/create-dirs diffs-path) - (println "Clay: Creating diff file" diff-file "& old/new") - (when diff-print-fn - (spit diff-file - (with-out-str - (print-diffs diff-print-fn note-diffs)))) + (if (some not-empty (ddiff/minimize note-diffs)) + (when diff-print-fn + (println "Clay: Creating diff file" diff-file "& old/new") + (spit diff-file (with-out-str + (print-diffs diff-print-fn note-diffs)))) + (do (println "Clay: Creating no-diff file" no-diff-file "& old/new") + (spit no-diff-file "no difference"))) (spit old-file (with-out-str (print-fn old))) (spit new-file (with-out-str (print-fn new))) + (println "Clay: Copying latest diff files to" (str diffs-base-path)) (doseq [file (fs/list-dir diffs-base-path #(fs/regular-file? % {:nofollow-links true}))] (fs/delete file)) @@ -58,29 +70,32 @@ (assert (or to-files to-repl) "Please pick an output option") (assert timestamp "Should be assoc'd to spec in scicloj.clay.v2.make/make!") (let [[old new] (pad-notes old new) - note-diffs (mapv ddiff/diff old new)] - (when (not-empty (ddiff/minimize note-diffs)) - (let [file-diff-print-fn (case to-files - :deep-diff2/full ddiff/pretty-print - :deep-diff2/minimal (comp ddiff/pretty-print - ddiff/minimize) - ;; No diff, we write old/new for to-files anyways - (:clojure/pprint nil) nil) - print-fn pp/pprint - repl-print-fn (case to-repl - :deep-diff2/full #(print-diffs ddiff/pretty-print - note-diffs) - :deep-diff2/minimal #(print-diffs (comp ddiff/pretty-print - ddiff/minimize) - note-diffs) - :clojure/pprint #(doseq [[old' new'] (map vector old new)] - (println "--------- old: ") - (print-fn old') - (println "--------- new: ") - (print-fn new')) - nil nil)] - (when to-repl - (println "Clay: Notes diff") - (repl-print-fn)) - (when to-files - (write-diff-files old new note-diffs file-diff-print-fn print-fn spec)))))) + note-diffs (mapv ddiff/diff old new) + file-diff-print-fn (case to-files + (:deep-diff2/full :deep-diff2/minimal) + (diff-print-fn to-files) + ;; No diff, but we always write old/new + ;; when to-files is specified + (:clojure/pprint nil) nil) + print-fn pp/pprint + repl-print-fn (case to-repl + (:deep-diff2/full :deep-diff2/minimal) + #(print-diffs (diff-print-fn to-repl) note-diffs) + :clojure/pprint #(doseq [[old* new*] (map vector old new)] + (println "--------- old: ") + (print-fn old*) + (println "--------- new: ") + (print-fn new*)) + nil nil)] + (when to-repl + (println "Clay: Notes diff start") + (let [diff (some not-empty (ddiff/minimize note-diffs))] + (when-not diff + (println "Clay: >>>> No difference!")) + (repl-print-fn) + ;; Add note on no difference at the end too when printing all data + (when (and (= to-repl :clojure/pprint) (not diff)) + (println "Clay: >>>> No difference!"))) + (println "Clay: Notes diff end")) + (when to-files + (write-diff-files old new note-diffs file-diff-print-fn print-fn spec)))) From 7c4c39047a5d5718714fd4b9fc2e60f3346ccd53 Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Sat, 29 Nov 2025 07:01:41 +0100 Subject: [PATCH 19/27] Attempt to dodge some datasets before they blow up diffing --- src/scicloj/clay/v2/util/diff.clj | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/scicloj/clay/v2/util/diff.clj b/src/scicloj/clay/v2/util/diff.clj index 61c43221..d1d55691 100644 --- a/src/scicloj/clay/v2/util/diff.clj +++ b/src/scicloj/clay/v2/util/diff.clj @@ -63,6 +63,30 @@ [(into [] (concat old (pad-to new old))) (into [] (concat new (pad-to old new)))])) +;; TODO clean up +(def only-eq-kinds + #{:kind/fn + :kind/table + :kind/dataset}) + +(defn- only-eq-kind-value [item] + (when (only-eq-kinds (:kind item)) + (:value item))) + +(defn- prep-notes [old new] + [(mapv (fn [old* new*] + (let [old-v (only-eq-kind-value old*) + new-v (only-eq-kind-value new*) + equal-v? (= old-v new-v)] + (cond-> old* old-v (assoc :value [(:kind old*) :value :eq equal-v?])))) + old new) + (mapv (fn [old* new*] + (let [old-v (only-eq-kind-value old*) + new-v (only-eq-kind-value new*) + equal-v? (= old-v new-v)] + (cond-> new* new-v (assoc :value [(:kind new*) :value :eq equal-v?])))) + old new)]) + (defn notes [old new & {:diff/keys [to-files to-repl timestamp] @@ -70,6 +94,7 @@ (assert (or to-files to-repl) "Please pick an output option") (assert timestamp "Should be assoc'd to spec in scicloj.clay.v2.make/make!") (let [[old new] (pad-notes old new) + [old new] (prep-notes old new) note-diffs (mapv ddiff/diff old new) file-diff-print-fn (case to-files (:deep-diff2/full :deep-diff2/minimal) From 0bc482ae1047a91956ae4c5582e85c092b98460a Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Sun, 30 Nov 2025 21:16:57 +0100 Subject: [PATCH 20/27] Diff only clojure- and java-datastructures for now --- src/scicloj/clay/v2/util/diff.clj | 30 ++------------------ src/scicloj/clay/v2/util/diff/prep.clj | 38 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 27 deletions(-) create mode 100644 src/scicloj/clay/v2/util/diff/prep.clj diff --git a/src/scicloj/clay/v2/util/diff.clj b/src/scicloj/clay/v2/util/diff.clj index d1d55691..9267c388 100644 --- a/src/scicloj/clay/v2/util/diff.clj +++ b/src/scicloj/clay/v2/util/diff.clj @@ -2,7 +2,8 @@ (:require [clojure.string :as str] [clojure.pprint :as pp] [babashka.fs :as fs] - [lambdaisland.deep-diff2 :as ddiff])) + [lambdaisland.deep-diff2 :as ddiff] + [scicloj.clay.v2.util.diff.prep :as prep-diff])) (defn- print-diffs [diff-print-fn note-diffs] (doseq [diff note-diffs] @@ -63,38 +64,13 @@ [(into [] (concat old (pad-to new old))) (into [] (concat new (pad-to old new)))])) -;; TODO clean up -(def only-eq-kinds - #{:kind/fn - :kind/table - :kind/dataset}) - -(defn- only-eq-kind-value [item] - (when (only-eq-kinds (:kind item)) - (:value item))) - -(defn- prep-notes [old new] - [(mapv (fn [old* new*] - (let [old-v (only-eq-kind-value old*) - new-v (only-eq-kind-value new*) - equal-v? (= old-v new-v)] - (cond-> old* old-v (assoc :value [(:kind old*) :value :eq equal-v?])))) - old new) - (mapv (fn [old* new*] - (let [old-v (only-eq-kind-value old*) - new-v (only-eq-kind-value new*) - equal-v? (= old-v new-v)] - (cond-> new* new-v (assoc :value [(:kind new*) :value :eq equal-v?])))) - old new)]) - (defn notes [old new & {:diff/keys [to-files to-repl timestamp] :as spec}] (assert (or to-files to-repl) "Please pick an output option") (assert timestamp "Should be assoc'd to spec in scicloj.clay.v2.make/make!") - (let [[old new] (pad-notes old new) - [old new] (prep-notes old new) + (let [[old new] (prep-diff/replace-undiffable (pad-notes old new)) note-diffs (mapv ddiff/diff old new) file-diff-print-fn (case to-files (:deep-diff2/full :deep-diff2/minimal) diff --git a/src/scicloj/clay/v2/util/diff/prep.clj b/src/scicloj/clay/v2/util/diff/prep.clj new file mode 100644 index 00000000..48d55a0a --- /dev/null +++ b/src/scicloj/clay/v2/util/diff/prep.clj @@ -0,0 +1,38 @@ +(ns scicloj.clay.v2.util.diff.prep + (:require [clojure.walk :as walk] + [clojure.string :as str])) + +(defprotocol DiffableBaseType + (diffable-base-type? [this])) + +(extend-protocol DiffableBaseType + java.util.Set + (diffable-base-type? [_] true) + java.util.Map + (diffable-base-type? [_] true) + java.util.List + (diffable-base-type? [_] true) + clojure.lang.IPersistentVector + (diffable-base-type? [_] true)) + +(defn type-str [x] + (-> x type pr-str)) + +(defn diffable-type? [x] + (let [x-type-str (type-str x)] + (some (partial String/.startsWith x-type-str) + ["java.util" + "clojure.lang"]))) + +(defn describe-type [t] + (type-str t)) + +(defn replace-undiffable* [x] + (cond (fn? x) ::fn + (and (satisfies? DiffableBaseType x) + (diffable-base-type? x) + (not (diffable-type? x))) (describe-type x) + :else x)) + +(defn replace-undiffable [notes] + (walk/prewalk replace-undiffable* notes)) From 16a65ebe049496028a1bdb4c7a185c341a24c1f1 Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Mon, 1 Dec 2025 00:56:34 +0100 Subject: [PATCH 21/27] vector is list --- src/scicloj/clay/v2/util/diff/prep.clj | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/scicloj/clay/v2/util/diff/prep.clj b/src/scicloj/clay/v2/util/diff/prep.clj index 48d55a0a..c0051a74 100644 --- a/src/scicloj/clay/v2/util/diff/prep.clj +++ b/src/scicloj/clay/v2/util/diff/prep.clj @@ -11,8 +11,6 @@ java.util.Map (diffable-base-type? [_] true) java.util.List - (diffable-base-type? [_] true) - clojure.lang.IPersistentVector (diffable-base-type? [_] true)) (defn type-str [x] From a7a0274db91c04e1fd7f7ba5267ecb67f7d4d9ff Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Mon, 1 Dec 2025 00:59:32 +0100 Subject: [PATCH 22/27] Added ReplacedValue type for diffing, wraps replaced value --- src/scicloj/clay/v2/util/diff/prep.clj | 37 +++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/src/scicloj/clay/v2/util/diff/prep.clj b/src/scicloj/clay/v2/util/diff/prep.clj index c0051a74..86194684 100644 --- a/src/scicloj/clay/v2/util/diff/prep.clj +++ b/src/scicloj/clay/v2/util/diff/prep.clj @@ -16,20 +16,49 @@ (defn type-str [x] (-> x type pr-str)) +(defn describe-type [t] + (type-str t)) + +(declare replaced-value=) + +(defprotocol PReplacedValue + (value? [this])) + +(deftype ReplacedValue [value value-type] + PReplacedValue + (value? [_] + (= (describe-type value) value-type)) + + Object + (equals [this other] + (or (not (value? this)) + (replaced-value= this other))) + (hashCode [this] + (hash-combine (hash value-type) + (if (value? this) + (hash value) + value))) + (toString [this] + (pr-str (.hashCode this) + value-type))) + +(defn replaced-value= [^ReplacedValue this other] + (and (instance? ReplacedValue this) + (instance? ReplacedValue other) + (= (.-value-type this) (.-value-type ^ReplacedValue other)) + (= (.-value this) (.-value ^ReplacedValue other)))) + (defn diffable-type? [x] (let [x-type-str (type-str x)] (some (partial String/.startsWith x-type-str) ["java.util" "clojure.lang"]))) -(defn describe-type [t] - (type-str t)) - (defn replace-undiffable* [x] (cond (fn? x) ::fn (and (satisfies? DiffableBaseType x) (diffable-base-type? x) - (not (diffable-type? x))) (describe-type x) + (not (diffable-type? x))) (->ReplacedValue x (describe-type x)) :else x)) (defn replace-undiffable [notes] From 008e89ac5c617918b45bdf9ea0f8540cff74841b Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Mon, 1 Dec 2025 01:09:47 +0100 Subject: [PATCH 23/27] Just replace unsupported collections for now --- src/scicloj/clay/v2/util/diff/prep.clj | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/scicloj/clay/v2/util/diff/prep.clj b/src/scicloj/clay/v2/util/diff/prep.clj index 86194684..13b7d309 100644 --- a/src/scicloj/clay/v2/util/diff/prep.clj +++ b/src/scicloj/clay/v2/util/diff/prep.clj @@ -55,11 +55,11 @@ "clojure.lang"]))) (defn replace-undiffable* [x] - (cond (fn? x) ::fn - (and (satisfies? DiffableBaseType x) - (diffable-base-type? x) - (not (diffable-type? x))) (->ReplacedValue x (describe-type x)) - :else x)) + (if (and (satisfies? DiffableBaseType x) + (diffable-base-type? x) + (not (diffable-type? x))) + (->ReplacedValue x (describe-type x)) + x)) (defn replace-undiffable [notes] (walk/prewalk replace-undiffable* notes)) From 24fcbad91a1509348f601856e27a993e383d458e Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Mon, 1 Dec 2025 05:45:07 +0100 Subject: [PATCH 24/27] Fix folder to delete in should exist first --- src/scicloj/clay/v2/util/diff.clj | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/scicloj/clay/v2/util/diff.clj b/src/scicloj/clay/v2/util/diff.clj index 9267c388..d4513355 100644 --- a/src/scicloj/clay/v2/util/diff.clj +++ b/src/scicloj/clay/v2/util/diff.clj @@ -32,16 +32,16 @@ no-diff-file (str diff-base-file ".no-diff") diff-file (str diff-base-file ".diff.edn") old-file (str diff-base-file ".old.edn") - new-file (str diff-base-file ".new.edn") - diff-dirs (->> (fs/list-dir diffs-base-path - #(fs/directory? % {:nofollow-links true})) - (sort-by fs/last-modified-time))] - (when (number? keep-dirs) - (doseq [dir (take (max 0 (inc (- (count diff-dirs) keep-dirs))) - diff-dirs)] - (when (= (fs/parent dir) diffs-base-path) - (fs/delete-tree dir)))) + new-file (str diff-base-file ".new.edn")] (fs/create-dirs diffs-path) + (when (number? keep-dirs) + (let [diff-dirs (->> (fs/list-dir diffs-base-path + #(fs/directory? % {:nofollow-links true})) + (sort-by fs/last-modified-time))] + (doseq [dir (take (max 0 (inc (- (count diff-dirs) keep-dirs))) + diff-dirs)] + (when (= (fs/parent dir) diffs-base-path) + (fs/delete-tree dir))))) (if (some not-empty (ddiff/minimize note-diffs)) (when diff-print-fn (println "Clay: Creating diff file" diff-file "& old/new") From 5108531c7d1c07214e4546ba55ea81aa583d5a9e Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Tue, 2 Dec 2025 00:35:46 +0100 Subject: [PATCH 25/27] Fixed future handler in deep-diff2 printer But maybe we should not diff these. --- deps.edn | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deps.edn b/deps.edn index 1a809c73..28a6951f 100644 --- a/deps.edn +++ b/deps.edn @@ -4,7 +4,8 @@ nrepl/nrepl {:mvn/version "1.3.1"} com.cnuernber/charred {:mvn/version "1.037"} read-kinds/read-kinds {:local/root "../read-kinds"} - lambdaisland/deep-diff2 {:mvn/version "2.12.219"} + ;; TODO back to lambdaisland/deep-diff2 once fix merged + io.github.onbreath/deep-diff2 {:git/sha "1f969521b68ce9dd9feed9b51a99a4569482b6ad"} carocad/parcera {:mvn/version "0.11.6"} org.antlr/antlr4-runtime {:mvn/version "4.7.1"} http-kit/http-kit {:mvn/version "2.8.0"} From a88d93cc5c77bee2b85a7e210463b9a814e0dc74 Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Tue, 2 Dec 2025 03:57:57 +0100 Subject: [PATCH 26/27] Make old and new comments equal for diff --- src/scicloj/clay/v2/notebook.clj | 34 ++++++++++++++++++++++------ src/scicloj/clay/v2/read.clj | 38 +++++++++++++++++++------------- 2 files changed, 50 insertions(+), 22 deletions(-) diff --git a/src/scicloj/clay/v2/notebook.clj b/src/scicloj/clay/v2/notebook.clj index 6f6be87d..ea4e538f 100644 --- a/src/scicloj/clay/v2/notebook.clj +++ b/src/scicloj/clay/v2/notebook.clj @@ -461,17 +461,35 @@ "seconds") result#)) -(defn notes-dissoc-old [notes] +(defn ->old-comment [note] + (let [comment-item (-> note :code notebook-old/comment->item)] + (-> note + (assoc :value (str/replace (:md comment-item) + ;; TODO stripping extra space added + ;; in front of headline Do we want + ;; to add this for read-kinds? + #"\n#" "#") + :kind :kind/md) + (dissoc :code :comment? :region)))) + +(defn ->old-notes-approx [notes] (->> notes (into [] (map #(-> % + (cond-> (and (:comment? %) + (:code %)) + ->old-comment) (dissoc :gen)))))) -(defn notes-dissoc-new [notes] +(defn ->new-notes-approx [notes] (->> notes (into [] (map #(-> % - (dissoc :line :column)))))) + (dissoc :line :column) + (cond-> (-> % :narrowed nil?) + (dissoc :narrowed) + (-> % :narrower nil?) + (dissoc :narrower))))))) (defn spec-notes [{:as spec :keys [pprint-margin ns-form full-source-path] @@ -481,15 +499,17 @@ *unchecked-math* *unchecked-math* pp/*print-right-margin* pprint-margin] (let [old (-> (notebook-old/spec-notes spec) - (notes-dissoc-old)) - new (-> (relevant-notes spec) + (->old-notes-approx)) + ;; TODO Comment blocks equal to old ones + new (-> (assoc spec :collapse-comments-ws? true) + (relevant-notes) (complete-notes spec) + (->new-notes-approx) ;; TODO Read kinds should be doing this #_(with-out-err-captured) (log-time (str "Evaluated notebook with read-kinds " (or (some-> ns-form second name) - (some-> full-source-path fs/file-name)))) - (notes-dissoc-new))] + (some-> full-source-path fs/file-name)))))] ;; We can print the plain new and old notes.. #_(diff/notes old new :diff/to-repl :clojure/pprint diff --git a/src/scicloj/clay/v2/read.clj b/src/scicloj/clay/v2/read.clj index b3da7677..f5e83e62 100644 --- a/src/scicloj/clay/v2/read.clj +++ b/src/scicloj/clay/v2/read.clj @@ -29,23 +29,32 @@ (-> form first (= 'ns))))) first)) -(defn collapse-comments-ws [notes] - (let [collapse (comp #{:kind/whitespace :kind/comment} :kind) - comment? (comp #{:kind/comment} :kind)] - (->> notes - (partition-by (comp boolean collapse)) - (mapcat - (fn [notes*] - (if (some comment? notes*) - [{:value (str/join (map :value - notes*)) - :kind :kind/comment}] - notes*)))))) +(defn collapse-comments-ws [collapse-comments-ws? notes] + (if collapse-comments-ws? + (let [collapse (comp #{:kind/whitespace :kind/comment} :kind) + comment? (comp #{:kind/comment} :kind)] + (->> notes + (partition-by (comp boolean collapse)) + (mapcat + (fn [notes*] + (if (some comment? notes*) + ;; TODO Pulling in all comments and whitespace + ;; This is only done to easily get equality with old comments + [{:value (let [comment* (->> notes* + (map #(get % :value (:code %))) + str/join)] + (-> comment* + (str/replace #"^\s+" "") + (str/trim-newline))) + :kind :kind/comment}] + notes*))))) + notes)) ;; TODO keep this or something like it (defn ->notes [{:keys [single-form single-value - code]}] + code + collapse-comments-ws?]}] (cond single-value (conj (when code [{:form (read-ns-form code)}]) {:value single-value}) @@ -55,8 +64,7 @@ {:form single-form}) :else (->> code (read/read-string-all) - ;; TODO maybe optional for diffing - collapse-comments-ws + (collapse-comments-ws collapse-comments-ws?) (into [] notes/notebook-xform)))) ;; TODO: Not needed? read-kinds has a safe-notes wrapper already... From 41c81f2df17ea116de06d4e6556b5428d909e43b Mon Sep 17 00:00:00 2001 From: onbreath <65270097+onbreath@users.noreply.github.com> Date: Thu, 4 Dec 2025 01:01:01 +0100 Subject: [PATCH 27/27] read/eval split in read-kinds --- src/scicloj/clay/v2/read.clj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scicloj/clay/v2/read.clj b/src/scicloj/clay/v2/read.clj index f5e83e62..7c86b5c5 100644 --- a/src/scicloj/clay/v2/read.clj +++ b/src/scicloj/clay/v2/read.clj @@ -64,6 +64,7 @@ {:form single-form}) :else (->> code (read/read-string-all) + (read/eval-ast) (collapse-comments-ws collapse-comments-ws?) (into [] notes/notebook-xform))))