refac

README.md

@@ -158,37 +158,36 @@ File downloads and proxy traffic stream in chunks. Uploads and generated ZIP/PDF
 files use temporary files, removed on completion or cancellation. Command logs
 rotate on disk without reading the whole log into memory.
 
-Text and log reads return bounded pages. When `truncated` is true, continue text
-reads from `next_line` or log reads from `next_offset`. Explicit log `tail` reads
-return the newest output and can skip older entries. Download a file through
-`/files/view` when you need its complete original bytes.
+Text reads preserve the original `path`, `total_lines`, and `content` response,
+including complete long lines and the requested line range. The response is
+built on disk and streamed, without loading the entire source into a list of
+lines. Log reads return bounded pages; continue from `next_offset` when
+`truncated` is true. Explicit log `tail` reads return the newest output and can
+skip older entries.
 
 These settings accept positive integers, through TOML keys or the corresponding
 `OPEN_TERMINAL_` environment variable (for example, `OPEN_TERMINAL_MAX_RESPONSE_BYTES`):
 
 | TOML key | Default | Scope |
 |---|---:|---|
-| `max_response_bytes` | 1048576 | Text/log content per page and notebook output budget |
-| `max_request_bytes` | 2097152 | Ordinary HTTP request bodies |
-| `max_transfer_bytes` | 1073741824 | Upload/proxy request bodies and uncompressed ZIP input |
+| `max_response_bytes` | 1048576 | Log content per page and notebook output budget |
+| `max_request_bytes` | 2097152 | HTTP request bodies outside file/skill APIs |
+| `max_transfer_bytes` | 1073741824 | Proxy request bodies |
 | `max_command_seconds` | 3600 | Background command lifetime; separate from the output wait timeout |
-| `worker_memory_bytes` | 536870912 | Linux address-space limit for document workers only |
 
 Commands, notebook sessions, and HTTP requests have no global concurrency cap.
-Oversized inputs return 413. JSON bodies are also checked for excessive nesting/object counts
-before parsing. Background commands that exceed their deadline stop with an
+Outside file/skill APIs, oversized inputs return 413 and JSON bodies are
+checked for excessive nesting/object counts before parsing. Background commands that exceed their deadline stop with an
 `error` in their status response. Log I/O failures are reported there too; output
 is still drained and the command's completion state is finalized.
 
-Directory scans examine at most 10,000 entries; content search scans at most
-32 MiB, returns bounded snippets, and skips lines longer than 64 KiB. Narrow the
-search path when it exceeds these budgets. Text editing is limited to the text
-response budget and 50,000 lines. Text reads reject individual lines longer than
-64 KiB. Document extraction allows up to 16 MiB of input/expanded ZIP content,
-2 million extracted characters, and 100,000 spreadsheet cells. Two document
-workers may run concurrently, with a 60-second deadline and bounded output;
-additional document work returns HTTP 429.
-Limits are per server process; additional server workers multiply them.
+Searches preserve complete matching lines, ignore rules, ranking, and pagination.
+Git/ripgrep listings and ranked content matches use temporary disk storage rather
+than collecting the whole search in memory. File reads, edits, uploads, archives,
+and document extraction have no added size, line-count, cell-count, or scan caps.
+Document extraction writes text to a temporary file in a separate worker, with
+cleanup on completion or cancellation. File comparison retains its pre-existing
+50 MiB input, 2 million characters, 50,000 lines, and 60-second limits.
 
 Notebook kernels are stopped on cancelled execution, excessive output, or failed
 startup, and idle sessions expire after 30 minutes. Ordinary Python cell errors
@@ -207,9 +206,8 @@ The host/container supervisor must delegate that cgroup and grant access to the
 worker users; Open Terminal does not create or modify host cgroups. Without it,
 application budgets do **not** bound arbitrary child allocations or forks.
 A memory limit on the entire container alone can still kill the API.
-The document address-space limit is Linux-only; macOS and Windows retain the
-application budgets but require platform isolation for hard workload memory
-limits.
+Hard workload memory isolation requires support from the deployment platform;
+application streaming and cleanup cannot constrain arbitrary user programs.
 
 ### File Browser Root
 

open_terminal/main.py

@@ -35,7 +35,7 @@ from pydantic import BaseModel, Field
 
 from open_terminal.env import API_KEY, BINARY_FILE_MIME_PREFIXES, CORS_ALLOWED_ORIGINS, ENABLE_NOTEBOOKS, ENABLE_SYSTEM_PROMPT, ENABLE_TERMINAL, EXECUTE_DESCRIPTION, EXECUTE_TIMEOUT, FILE_BROWSER_ROOT, LOG_DIR, MAX_TERMINAL_SESSIONS, MULTI_USER, OPEN_TERMINAL_INFO, PROCESS_LOG_RETENTION, SESSION_CWD_TTL, SYSTEM_PROMPT, TERMINAL_TERM
 from open_terminal.utils.runner import PipeRunner, ProcessRunner, create_runner, read_pty, write_pty, PtyRunner
-from open_terminal.utils.fs import UserFS, TemporaryFileResponse, text_file_response
+from open_terminal.utils.fs import UserFS, TemporaryFileResponse, text_file_response, line_range_offsets
 from open_terminal.utils.limits import (CHUNK_SIZE, ResourceLimitError, ResourceLimitsMiddleware, run_io, run_worker, worker_command)
 from open_terminal.utils.file_compare import CompareRequest, run_comparison
 from open_terminal.utils.search import search_entries, walk_entries, matching_files, ranked_page
@@ -953,10 +953,8 @@ async def replace_file_content(http_request: Request, request: ReplaceRequest, f
 
     for chunk in request.replacements:
         if chunk.start_line or chunk.end_line:
-            lines = content.splitlines(keepends=True)
-            start = (chunk.start_line or 1) - 1
-            end = chunk.end_line or len(lines)
-            search_region = "".join(lines[start:end])
+            start, end = line_range_offsets(content, chunk.start_line, chunk.end_line)
+            search_region = content[start:end]
         else:
             search_region = content
 
@@ -974,8 +972,7 @@ async def replace_file_content(http_request: Request, request: ReplaceRequest, f
 
         if chunk.start_line or chunk.end_line:
             new_region = search_region.replace(chunk.target, chunk.replacement)
-            lines[start:end] = [new_region]
-            content = "".join(lines)
+            content = content[:start] + new_region + content[max(start, end):]
         else:
             content = content.replace(chunk.target, chunk.replacement)
 
@@ -1165,7 +1162,7 @@ async def match_files(
                         eligible = rg_matches(entry_path) if rg_matches is not None else size <= MAX_CONTENT_SEARCH_FILE_SIZE
                         if eligible:
                             with open(entry_path, "rb") as source:
-                                binary = b"\0" in source.read(8192)
+                                binary = rg_matches is None and b"\0" in source.read(8192)
                             for number, text in (() if binary else content_lines(entry_path)):
                                 index = text.lower().find(query_lower)
                                 if index >= 0:
@@ -1220,7 +1217,7 @@ async def search_files(
     def _search_sync():
         query_lower = query.strip().lower()
         def candidates():
-            for full_path, entry_type in search_entries(fs, target, show_hidden):
+            for full_path, entry_type in search_entries(fs, target, show_hidden, git_files_as_files=True):
                 if type != "any" and entry_type != type:
                     continue
                 name = os.path.basename(full_path)

open_terminal/utils/documents.py

@@ -17,9 +17,9 @@ def text_extractor(separator="\n"):
         def extract(*args, **kwargs):
             return separator.join(function(*args, **kwargs))
 
-        def write_to(path, output):
+        def write_to(path, output, **kwargs):
             first = True
-            for part in function(path):
+            for part in function(path, **kwargs):
                 if not first:
                     output.write(separator)
                 output.write(part)
@@ -315,7 +315,7 @@ if __name__ == "__main__":
         path = payload["path"]
         mime = mimetypes.guess_type(path)[0]
         extractor = next(fn for kind, suffix, fn in EXTRACTORS if kind and kind == mime or suffix and path.lower().endswith(suffix))
-        with open(payload["output"], "w", encoding="utf-8") as output:
+        with open(payload["output"], "w", encoding="utf-8", newline="") as output:
             if hasattr(extractor, "write_to"):
                 extractor.write_to(path, output)
             else:

open_terminal/utils/file_compare.py

@@ -1,6 +1,7 @@
 """Read-only text comparison. Workers receive structured input on stdin."""
 
 import difflib
+import io
 import json
 import mimetypes
 import os
@@ -26,6 +27,14 @@ MAX_TEXT_CHARS = 2_000_000
 MAX_LINES = 50_000
 
 
+class ComparisonText(io.StringIO):
+    def write(self, text):
+        # CRLF normalization can halve the length; enforce the exact limit below.
+        if self.tell() + len(text) > 2 * MAX_TEXT_CHARS:
+            raise ValueError("Extracted text exceeds the comparison limit (2 million characters / 50,000 lines). Nothing was truncated.")
+        return super().write(text)
+
+
 class CompareRequest(BaseModel):
     original: str = Field(min_length=1)
     revised: str = Field(min_length=1)
@@ -48,7 +57,12 @@ def extract(path):
             None,
         )
         if extractor:
-            if extractor in (extract_pdf, extract_epub):
+            if hasattr(extractor, "write_to"):
+                output = ComparisonText()
+                options = {"strict": True} if extractor in (extract_pdf, extract_epub) else {}
+                extractor.write_to(path, output, **options)
+                text = output.getvalue()
+            elif extractor in (extract_pdf, extract_epub):
                 text = extractor(path, strict=True)
             else:
                 text = extractor(path)
@@ -92,11 +106,13 @@ def extract(path):
         text = text.replace("\r\n", "\n").replace("\r", "\n")
         if len(text) > MAX_TEXT_CHARS:
             raise ValueError("Extracted text exceeds the comparison limit.")
-        lines = text.splitlines()
-        if len(lines) > MAX_LINES:
+        line_breaks = "\n\v\f\x1c\x1d\x1e\x85\u2028\u2029"
+        line_count = sum(1 for _ in re.finditer("[" + line_breaks + "]", text)) + bool(text and text[-1] not in line_breaks)
+        if line_count > MAX_LINES:
             raise ValueError(
                 "Extracted text exceeds the comparison limit (2 million characters / 50,000 lines). Nothing was truncated."
             )
+        lines = text.splitlines()
         return {"path": path, "name": name, "notices": notices}, lines
     except Exception as exc:
         raise ValueError(f"{name}: {exc}") from exc

open_terminal/utils/fs.py

@@ -43,6 +43,22 @@ class TemporaryFileResponse(FileResponse):
 _LINE_BREAK = re.compile(r"\r\n|[\r\n\v\f\x1c-\x1e\x85\u2028\u2029]")
 
 
+def line_range_offsets(text, start_line=None, end_line=None):
+    start_boundary = (start_line or 1) - 1
+    start = 0 if start_boundary == 0 else len(text)
+    end = len(text)
+    last_boundary = max(start_boundary, end_line or 0)
+    if last_boundary:
+        for number, match in enumerate(_LINE_BREAK.finditer(text), 1):
+            if number == start_boundary:
+                start = match.end()
+            if number == end_line:
+                end = match.end()
+            if number >= last_boundary:
+                break
+    return start, end
+
+
 def text_chunks(source):
     """Yield splitlines-compatible line numbers and bounded pieces of text."""
     number = 1

open_terminal/utils/limits.py

@@ -102,13 +102,8 @@ class ResourceLimitsMiddleware:
         await self.app(scope, limited_receive, send)
 
 
-def worker_command(command: list[str], *, memory_limit: int = 0) -> list[str]:
-    """Apply workload isolation before exec, without unsafe preexec_fn.
-
-    Address-space limits suit our document workers. General runtimes (Node,
-    JVM, scientific Python) reserve large virtual ranges and need a cgroup
-    budget on actual memory instead.
-    """
+def worker_command(command: list[str]) -> list[str]:
+    """Join the configured workload cgroup before exec, without preexec_fn."""
     if os.name == "nt":
         return command
     cgroup = os.environ.get("OPEN_TERMINAL_WORKLOAD_CGROUP", "")
@@ -116,9 +111,9 @@ def worker_command(command: list[str], *, memory_limit: int = 0) -> list[str]:
         path = Path(cgroup)
         if (path / "memory.max").read_text().strip() == "max":
             raise ValueError("OPEN_TERMINAL_WORKLOAD_CGROUP must have a finite memory.max")
-    if not memory_limit and not cgroup:
+    if not cgroup:
         return command
-    return [sys.executable, str(Path(__file__).with_name("worker.py")), str(memory_limit), cgroup, *command]
+    return [sys.executable, str(Path(__file__).with_name("worker.py")), cgroup, *command]
 
 
 async def run_worker(command, payload=None, *, request=None, timeout=60, **options):

open_terminal/utils/search.py

@@ -43,7 +43,7 @@ def disk_database():
             db.close()
 
 
-def search_entries(fs, root, show_hidden=False):
+def search_entries(fs, root, show_hidden=False, *, git_files_as_files=False):
     with tempfile.TemporaryFile() as output:
         try:
             result = subprocess.run(
@@ -65,11 +65,13 @@ def search_entries(fs, root, show_hidden=False):
                 path = os.path.normpath(os.path.join(root, rel))
                 if not fs.is_path_allowed(path):
                     continue
+                leaf = path
                 while path != root and path.startswith(root.rstrip(os.sep) + os.sep):
                     if fs.is_path_allowed(path):
                         cursor = db.execute("INSERT OR IGNORE INTO entries VALUES (?)", (os.fsencode(path),))
                         if cursor.rowcount:
-                            yield path, "directory" if os.path.isdir(path) else "file"
+                            kind = "file" if git_files_as_files and path == leaf else "directory" if os.path.isdir(path) else "file"
+                            yield path, kind
                     path = os.path.dirname(path)
 
 

open_terminal/utils/worker.py

@@ -1,23 +1,12 @@
-"""Small exec launcher. Limits are inherited by the command and its children."""
+"""Exec launcher for a supervisor-provided workload cgroup."""
 import os
-import resource
 import sys
 
 
 def main():
-    memory, cgroup, *command = sys.argv[1:]
-    if cgroup:
-        with open(os.path.join(cgroup, "cgroup.procs"), "w") as stream:
-            stream.write(str(os.getpid()))
-    # RLIMIT_AS is enforced on Linux. Aggregate process-tree limits require a
-    # delegated cgroup; per-process limits alone cannot contain arbitrary forks.
-    if sys.platform == "linux" and int(memory):
-        limit = int(memory)
-        _, hard = resource.getrlimit(resource.RLIMIT_AS)
-        if hard != resource.RLIM_INFINITY:
-            limit = min(limit, hard)
-        resource.setrlimit(resource.RLIMIT_AS, (limit, limit))
-    resource.setrlimit(resource.RLIMIT_CORE, (0, 0))
+    cgroup, *command = sys.argv[1:]
+    with open(os.path.join(cgroup, "cgroup.procs"), "w") as stream:
+        stream.write(str(os.getpid()))
     os.execvp(command[0], command)