patch generation stable again

4 files changed, 138 insertions, 148 deletions

diff --git a/patchtree/cli.py b/patchtree/cli.py
index 6e24bb4..f127d1c 100644
--- a/patchtree/cli.py
+++ b/patchtree/cli.py
@@ -124,7 +124,7 @@ def main():
     log_add_level_name(CRITICAL, "CRT")
     log_config(
         stream=stderr,
-        level=INFO,
+        level=WARNING,
         format="[%(levelname)s] %(message)s",
     )
 
@@ -133,7 +133,13 @@ def main():
     context = parse_arguments(config)
 
     patch = context.make_patch()
-    print(patch, end="")
+
+    if not context.in_place:
+        context.output.write(patch)
+        context.output.flush()
+        context.output.close()
+    else:
+        raise NotImplementedError("TODO")
 
     return 0
 
diff --git a/patchtree/context.py b/patchtree/context.py
index 7a66b1a..65f9568 100644
--- a/patchtree/context.py
+++ b/patchtree/context.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 from dataclasses import dataclass
+from io import StringIO
 from typing import TYPE_CHECKING, IO, Any, TextIO, cast
 
 from frontmatter.default_handlers import YAMLHandler
@@ -17,11 +18,13 @@ from zipfile import ZipFile
 from stat import S_IFDIR, S_IFREG
 from logging import Logger, getLogger as get_logger
 
+from yaml.events import DocumentEndEvent
 from yaml.nodes import MappingNode, ScalarNode
 
 from patchtree.diff import File
 
 from .patchspec import (
+    DefaultInputSpec,
     FileInputSpec,
     LiteralInputSpec,
     PatchsetFileInputSpec,
@@ -90,7 +93,7 @@ class DiskFS(FS):
     """Implementation of :any:`FS` for a regular directory. Reads directly from the disk."""
 
     def __init__(self, target):
-        super(DiskFS, self).__init__(target)
+        super().__init__(target)
 
     def get_dir(self, dir):
         here = self.target.joinpath(dir)
@@ -131,7 +134,7 @@ class ZipFS(FS):
     """
 
     def __init__(self, target):
-        super(ZipFS, self).__init__(target)
+        super().__init__(target)
         self.zip = ZipFile(str(target))
         for info in self.zip.infolist():
             path = Path(info.filename)
@@ -192,71 +195,66 @@ class ZipFS(FS):
 
 
 class PatchspecLoader(yaml.Loader):
-    @dataclass
-    class Marshall:
-        """
-        This is a stupid solution to a stupid problem. PyYAML's load() method for some reason
-        doesn't accept a loader class instance but a class type. It also doesn't allow you to
-        pass user data through kwargs, so all of the context is marshalled through the input
-        stream parameter and unpacked by PatchspecLoader's __init__ method.
-        """
-
-        patch: File
-        input: Path
-        context: Context
-
+    patch: File
     input: Path
     context: Context
 
+    input_specs: list[ProcessInputSpec] = []
+
     @staticmethod
-    def tag_file_target(loader: PatchspecLoader, node: ScalarNode):
+    def tag_file_target(loader: PatchspecLoader, node: ScalarNode) -> TargetFileInputSpec:
         target_root = loader.input.parent.relative_to(loader.context.root)
-        return TargetFileInputSpec(
-            path=target_root.joinpath(Path(loader.construct_scalar(node))),
-        )
+        path = loader.construct_scalar(node)
+        if len(path) == 0:
+            path = loader.input.name
+        spec = TargetFileInputSpec(path=target_root.joinpath(Path(path)))
+        loader.input_specs.append(spec)
+        return spec
 
     @staticmethod
-    def tag_file_input(loader: PatchspecLoader, node: ScalarNode):
+    def tag_file_input(loader: PatchspecLoader, node: ScalarNode) -> PatchsetFileInputSpec:
         input_root = loader.input.parent
-        return PatchsetFileInputSpec(
-            path=input_root.joinpath(Path(loader.construct_scalar(node))),
-        )
+        path = loader.construct_scalar(node)
+        if len(path) == 0:
+            path = loader.input.name
+        spec = PatchsetFileInputSpec(path=input_root.joinpath(Path(path)))
+        loader.input_specs.append(spec)
+        return spec
 
     @staticmethod
-    def tag_patchspec(loader: PatchspecLoader, node: MappingNode):
+    def tag_patchspec(loader: PatchspecLoader, node: MappingNode) -> Target:
         data = loader.construct_mapping(node, deep=True)
-        return Target.from_spec(
+        target = Target.from_spec(
             loader.context, loader.input.relative_to(loader.context.root), data
         )
+        target.inputs += loader.input_specs
+        return target
+
+    def __init__(self, context: Context, patch: File, input: Path):
+        self.context = context
+        self.patch = patch
+        self.input = input
 
-    def __init__(self, marshall: Marshall):
-        super(PatchspecLoader, self).__init__(marshall.patch.get_str())
-        self.context = marshall.context
-        self.input = marshall.input
+        super().__init__(self.patch.get_str())
 
         self.add_constructor("!patchspec", self.tag_patchspec)
         self.add_constructor("!target", self.tag_file_target)
         self.add_constructor("!input", self.tag_file_input)
 
-    @staticmethod
-    def load(marshall: PatchspecLoader.Marshall) -> Any:
-        return yaml.load(cast(Any, marshall), Loader=PatchspecLoader)
-
-
-class PatchtreeYAMLHandler(YAMLHandler):
-    marshall: PatchspecLoader.Marshall
-
-    output: Target | None = None
-
-    def __init__(self, marshall: PatchspecLoader.Marshall):
-        super(PatchtreeYAMLHandler, self).__init__()
+    def parse(self) -> Target:
+        try:
+            data = self.get_data()
+            if not isinstance(data, Target):
+                raise Exception("provided yaml is not a patchspec")
 
-        self.marshall = marshall
+            # strip frontmatter from input content if it exists
+            events = yaml.parse(self.patch.get_str())
+            end = next(ev for ev in events if isinstance(ev, DocumentEndEvent))
+            self.patch.content = self.patch.get_str()[end.end_mark.index :].lstrip()
 
-    def load(self, fm, **kwargs):
-        # bypass fm.metadata because it is converted into a dict
-        self.output = PatchspecLoader.load(self.marshall)
-        return {}
+            return data
+        finally:
+            self.dispose()
 
 
 class Context:
@@ -292,6 +290,8 @@ class Context:
 
     is_empty: bool = False
 
+    output: IO
+
     def __init__(self, config: Config, options: Namespace):
         self.config = config
         self.log = get_logger(self.__class__.__name__)
@@ -303,6 +303,7 @@ class Context:
         self.content = self.collect_targets(self.inputs)
 
         self.fs = self._get_fs(options.target)
+        self.output = self._get_output(options)
         self.header = config.header(config, self)
 
     def collect_inputs(self, options: Namespace) -> list[Path]:
@@ -346,46 +347,34 @@ class Context:
             # binary files can't be patchspecs
             target = Target(self, file)
 
-        marshall = PatchspecLoader.Marshall(
-            patch=patch,
-            input=input,
-            context=self,
-        )
-
         # if the input is a yaml file, try to load it
         if target is None and input.suffix in self.config.patchspec_extensions:
             try:
-                fake_marshall = marshall
-                fake_marshall.input = input.parent.joinpath(input.stem)
-                target = PatchspecLoader.load(fake_marshall)
+                loader = PatchspecLoader(self, patch, input.parent.joinpath(input.stem))
+                target = loader.parse()
+                self.log.info(f"found direct yaml patchspec: {input}")
             except Exception as e:
-                self.log.error(f"while parsing patchspec for {input}")
+                self.log.error(f"while parsing patchspec for {input}: {e}")
                 raise e
-            if not isinstance(target, Target):
-                target = None
 
-        # if the input contains frontmatter, try to load it
+        # try to load any frontmatter if we still don't have a target
         if target is None:
-            handler = PatchtreeYAMLHandler(marshall)
             try:
-                post = frontmatter.loads(patch.get_str(), handler=handler)
-                if not isinstance(handler.output, Target):
-                    raise Exception()
-                target = handler.output
-                # strip frontmatter from input content
-                patch.content = post.content
-            except:
+                loader = PatchspecLoader(self, patch, input)
+                target = loader.parse()
+                self.log.info(f"found frontmatter patchspec: {input}")
+            except Exception as e:
+                # exceptions while parsing frontmatter can be ignored silently since not all
+                # files will have them
                 target = None
 
         if target is None:
+            self.log.info(f"treating as literal input: {input}")
             target = Target(self, file)
 
-        target.file = target.file or str()
-        assert target.file is not None
+        target.patch = patch
 
-        for input in (
-            i.path for i in target.get_inputs() if isinstance(i, PatchsetFileInputSpec)
-        ):
+        for input in (i.path for i in target.inputs if isinstance(i, PatchsetFileInputSpec)):
             meta_inputs.append(input)
 
         return target
@@ -414,10 +403,10 @@ class Context:
         return sorted(targets.values(), key=lambda target: target.file)
 
     def get_file(self, spec: ProcessInputSpec) -> File:
-        if isinstance(spec, TargetFileInputSpec):
-            return self.fs.get_file(spec)
         if isinstance(spec, LiteralInputSpec):
             return File(content=spec.content, mode=MODE_FILE)
+        if isinstance(spec, TargetFileInputSpec):
+            return self.fs.get_file(spec)
         if isinstance(spec, PatchsetFileInputSpec):
             return File(
                 content=spec.path.read_bytes(),
@@ -444,15 +433,13 @@ class Context:
 
         raise Exception("cannot read `{target}'")
 
-    def get_output(self, options: Namespace) -> IO:
+    def _get_output(self, options: Namespace) -> IO:
         """
         Open the output stream, taking into account the --in-place and --out options.
 
         :returns: Output stream.
         """
         if options.in_place:
-            if options.out is not None:
-                self.log.warning("--out is ignored when using --in-place")
             return TextIO()
 
         if options.out is not None:
@@ -480,7 +467,6 @@ class Context:
     def make_patch(self) -> str:
         patch = ""
         for target in self.content:
-            self.log.info(f"writing patch for `{target.file}'")
             patch += target.write()
 
         self.is_empty = len(patch) == 0
diff --git a/patchtree/process.py b/patchtree/process.py
index a7871a8..d8450ad 100644
--- a/patchtree/process.py
+++ b/patchtree/process.py
@@ -1,4 +1,5 @@
 from __future__ import annotations
+from stat import S_IFREG
 from typing import TYPE_CHECKING, Any, Callable, Hashable, cast
 
 from tempfile import mkstemp
@@ -19,7 +20,6 @@ from .patchspec import (
 from .diff import File
 
 if TYPE_CHECKING:
-    from .context import Context
     from .target import Target
 
 
@@ -37,7 +37,7 @@ class Process:
     def __init__(self, target: Target):
         self.target = target
 
-    def transform(self, input: File) -> File:
+    def transform(self) -> File:
         """
         Perform the transformation of this processor.
 
@@ -45,10 +45,9 @@ class Process:
         """
         raise NotImplementedError()
 
-    @staticmethod
-    def from_spec(target: Target, data: dict[Hashable, Any]) -> Process:
+    @classmethod
+    def from_spec(cls, target: Target, data: dict[Hashable, Any]) -> Process:
         context = target.context
-        context.log.warning(repr(data))
         if "id" not in data:
             raise Exception("missing key 'id'")
         id = str(data["id"])
@@ -104,11 +103,13 @@ class Jinja2Process(Process):
     )
 
     def __init__(self, *args, **kwargs):
-        super(Jinja2Process, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
-    def transform(self, input):
+    def transform(self):
         template_vars = self.get_template_vars()
+        input = self.target.get_file(self.input_spec)
         input.content = self.environment.from_string(input.get_str()).render(**template_vars)
+
         return input
 
     def get_template_vars(self) -> dict[str, Any]:
@@ -122,9 +123,9 @@ class Jinja2Process(Process):
         """
         return {}
 
-    @staticmethod
-    def from_spec(target, data):
-        return Jinja2Process(target)
+    @classmethod
+    def from_spec(cls, target, data):
+        return cls(target)
 
 
 class CoccinelleProcess(Process):
@@ -133,12 +134,14 @@ class CoccinelleProcess(Process):
     """
 
     def __init__(self, *args, **kwargs):
-        super(CoccinelleProcess, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
+
+    def transform(self):
+        input = self.target.get_file(self.input_spec)
+        patch = self.target.get_file(self.target_spec)
 
-    def transform(self, input):
-        context = self.target.context
-        content_input = context.get_file(self.target_spec).get_str()
-        content_patch = input.get_str()
+        content_input = input.get_str()
+        content_patch = patch.get_str()
 
         # empty patch -> return input as-is (coccinelle gives errors in this case)
         if len(content_patch.strip()) == 0:
@@ -171,9 +174,17 @@ class CoccinelleProcess(Process):
 
         return input
 
-    @staticmethod
-    def from_spec(target, data):
-        return CoccinelleProcess(target)
+    @classmethod
+    def from_spec(cls, target, data):
+        process = cls(target)
+
+        if "input" not in data:
+            process.input_spec = TargetFileInputSpec(path=Path(target.file))
+
+        if "target" not in data:
+            process.target_spec = DefaultInputSpec()
+
+        return process
 
 
 class TouchProcess(Process):
@@ -183,19 +194,20 @@ class TouchProcess(Process):
 
     mode: int | None = None
 
-    def transform(self, input):
+    def transform(self):
+        input = self.target.get_file(self.input_spec)
         input.content = input.content or ""
         input.mode = self.mode or input.mode
         return input
 
-    @staticmethod
-    def from_spec(target, data):
-        process = TouchProcess(target)
+    @classmethod
+    def from_spec(cls, target, data):
+        process = cls(target)
 
         if "mode" in data:
             if not isinstance(data["mode"], int):
                 raise TypeError("invalid type of key 'mode'")
-            process.mode = data["mode"]
+            process.mode = data["mode"] | S_IFREG
             del data["mode"]
 
         return process
@@ -209,19 +221,21 @@ class ExecProcess(Process):
     cmd: list[str] = []
 
     def __init__(self, *args, **kwargs):
-        super(ExecProcess, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
-    def transform(self, input):
+    def transform(self):
         assert len(self.cmd) > 0
 
+        input = self.target.get_file(self.input_spec)
+
         proc = run(self.cmd, text=True, input=input.get_str(), capture_output=True, check=True)
         input.content = proc.stdout
 
         return input
 
-    @staticmethod
-    def from_spec(target, data):
-        process = ExecProcess(target)
+    @classmethod
+    def from_spec(cls, target, data):
+        process = cls(target)
 
         if "cmd" not in data:
             raise Exception("missing property `cmd'")
@@ -233,16 +247,6 @@ class ExecProcess(Process):
         return process
 
 
-# class MergeProcessorSpec(ProcessSpec):
-
-#     def _copy(self, spec: ProcessSpec):
-#         super(MergeProcessorSpec, self)._copy(spec)
-
-
-#     def __repr__(self):
-#         return f"{self.__class__.__name__}({self.input}, strategy={self.strategy.name})"
-
-
 class MergeProcess(Process):
     """
     Merge transformer.
@@ -254,9 +258,7 @@ class MergeProcess(Process):
 
         add_lines = set(lines_b) - set(lines_a)
 
-        b.content = "\n".join((*lines_a, *add_lines))
-
-        return b
+        return File(mode=a.mode, content="\n".join((*lines_a, *add_lines)))
 
     strategies: dict[str, Callable[[MergeProcess, File, File], File]] = {
         "ignore": merge_ignore,
@@ -265,17 +267,16 @@ class MergeProcess(Process):
     strategy: Callable[[MergeProcess, File, File], File] | None = None
 
     def __init__(self, *args, **kwargs):
-        super(MergeProcess, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
 
-    def transform(self, input):
-        context = self.target.context
-        a = input
-        b = context.get_file(self.target_spec)
+    def transform(self):
+        a = self.target.get_file(self.input_spec)
+        b = self.target.get_file(self.target_spec)
         assert self.strategy is not None
         return self.strategy(self, a, b)
 
-    @staticmethod
-    def from_spec(target, data):
+    @classmethod
+    def from_spec(cls, target, data):
         process = MergeProcess(target)
 
         if "strategy" not in data:
diff --git a/patchtree/target.py b/patchtree/target.py
index 859ed0f..eec6111 100644
--- a/patchtree/target.py
+++ b/patchtree/target.py
@@ -6,8 +6,6 @@ from pathlib import Path
 from .diff import Diff, File
 from .patchspec import (
     DefaultInputSpec,
-    FileInputSpec,
-    LiteralInputSpec,
     ProcessInputSpec,
     TargetFileInputSpec,
 )
@@ -15,7 +13,6 @@ from .process import Process
 
 if TYPE_CHECKING:
     from .context import Context
-    from .config import Config
 
 
 class Target:
@@ -40,6 +37,7 @@ class Target:
             data["processors"] = []
         if not isinstance(data["processors"], list):
             raise Exception("not a list: 'processors'")
+        target.processors = []
         for processor in data["processors"]:
             target.processors.append(Process.from_spec(target, processor))
 
@@ -49,31 +47,30 @@ class Target:
         self.context = context
         self.file = str(file)
 
-    def get_processors(self) -> list[Process]:
-        return self.processors
-
-    def get_inputs(self) -> list[ProcessInputSpec]:
-        return self.inputs
+    def get_file(self, spec: ProcessInputSpec) -> File:
+        if isinstance(spec, DefaultInputSpec):
+            return self.patch
+        return self.context.get_file(spec)
 
     def write(self) -> str:
         """
         Apply all processors, compare to the target and write the delta to :any:`Context.output`.
         """
-
-        config = self.context.config
-        assert self.file is not None
-        diff = Diff(config, self.file)
-
+        self.context.log.info(f"writing patch for `{self.file}'")
+
+        for i, processor in enumerate(self.processors):
+            try:
+                self.patch = processor.transform()
+            except Exception as e:
+                self.context.log.error(
+                    f"while running processor {i+1} ({processor.__class__.__name__}) for `{self.file}'"
+                )
+                raise e
+
+        diff = Diff(self.context.config, self.file)
         diff.a = self.context.get_file(TargetFileInputSpec(path=Path(self.file)))
         diff.b = self.patch
 
-        processors = self.get_processors()
-        for processor in processors:
-            input = diff.b
-            if not isinstance(processor.input_spec, DefaultInputSpec):
-                input = self.context.get_file(processor.input_spec)
-            diff.b = processor.transform(input)
-
         return diff.compare()
 
     def __repr__(self):