diff --git a/CHANGELOG.md b/CHANGELOG.md index a54cb677a..6b4fb494f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2862,16 +2862,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `sync.remote` silently left the remote live. Keys the write does not own, including dotted ones somebody else put there, are left exactly as they are. - Four things that used to succeed now report an error instead, because each one - produced a value no reader could see, or claimed to remove a value it had - left live. On `bd config set`: a key under a parent that already holds a value - (`sync: enabled`, then `bd config set sync.remote`), and a key in a file whose - top level is not a mapping. On `bd config unset`: a key inside a flow-style - mapping (`sync: {remote: "..."}`), which bd matches by line and so cannot - edit, and a key whose value is a block scalar (`remote: |`, in either - spelling), where commenting the key out would leave the body behind as a value - of its own. Unsetting a key that is not set remains a successful no-op in - every shape. + Six things that used to succeed now report an error instead, because each one + produced a value no reader could see, claimed to remove a value it had left + live, or damaged the rest of the file. On `bd config set`: a key under a parent + that already holds a value (`sync: enabled`, then `bd config set sync.remote`), + and a key in a file whose top level is not a mapping. On `bd config unset`: a + key inside a flow-style mapping (`sync: {remote: "..."}`, or a flat + `sync.remote` in a top level written that way), which bd matches by line and + so cannot edit; a key whose value is a block scalar (`remote: |`, in either + spelling), where commenting the key out would leave the body behind as a + value of its own; a key whose value starts on its line and wraps onto the + next (a `{...}` or `[...]` collection, a quoted string or a plain value spread + over lines, in either spelling), where commenting the key line out would + leave the rest of the value behind; and any other unset whose edit would + change more of the file than the key, such as a duplicate of the key that + wraps, the key in a later YAML document, or an anchor on its line that an + alias further down still uses, or that bd cannot check because the file + nests more than 64 levels deep. The same value written on one line still + unsets. Unsetting a key that is not set remains a successful no-op in every + shape, and no longer edits a line inside another value that only looks like + it. - **A self-hosted GitLab no longer re-creates its own issues on every push** ([#6735](https://github.com/gastownhall/beads/issues/6735)). The tracker decided whether a stored `external_ref` was one of its own by looking for diff --git a/internal/config/yaml_config.go b/internal/config/yaml_config.go index 9a2630c42..6bd220d8e 100644 --- a/internal/config/yaml_config.go +++ b/internal/config/yaml_config.go @@ -4,10 +4,12 @@ import ( "bufio" "errors" "fmt" + "io" "os" "os/exec" "path/filepath" "regexp" + "slices" "strconv" "strings" @@ -668,6 +670,9 @@ func UnsetUserYamlConfig(key string) (bool, error) { if newContent == normalized { return false, nil } + if err := collateralUnsetRefusal(normalized, newContent, key); err != nil { + return false, err + } // Preserve the owner-private 0600 posture every other user-global writer // uses (SetUserYamlConfig, setYamlConfigAtPath, the metrics bootstrap); @@ -774,6 +779,9 @@ func UnsetYamlConfig(key string) (bool, error) { if newContent == normalized { return false, nil } + if err := collateralUnsetRefusal(normalized, newContent, key); err != nil { + return false, err + } if err := os.WriteFile(configPath, []byte(newContent), 0600); err != nil { //nolint:gosec // configPath is validated return false, fmt.Errorf("failed to write config.yaml: %w", err) @@ -1200,6 +1208,16 @@ func commentOutYamlKey(content, key string) (string, error) { // long-standing behavior. content = strings.ReplaceAll(content, "\r\n", "\n") + // A key no document holds is not in the file, whatever its lines look + // like. The matcher below reads lines, so a line inside a quoted value or + // a list item can match the key. Commenting that line out edits a value + // nobody named, and refusing it fails the unset of a key that is not + // there, which has always been a successful no-op. So the file is left as + // it is, and both unset callers report that nothing changed. + if !yamlHoldsKey(content, key) { + return content, nil + } + if err := unsupportedUnsetShape(content, key); err != nil { return "", err } @@ -1414,22 +1432,26 @@ func (w *nestedKeyWalk) step(name string, indent int) bool { return false } -// unsupportedUnsetShape reports the two shapes named in the changelog — a key -// under a flow-style mapping, and a key whose value is a block scalar — in -// whichever spelling the file uses. Both used to be silent: the flow-style one -// reported success while the value stayed live, and the block scalar had its key -// line commented out while the body stayed behind as a value of its own. The set -// direction already refuses what it cannot write correctly, so unset says so too. +// unsupportedUnsetShape reports the shapes named in the changelog — a key under +// a flow-style mapping, a key whose value is a block scalar, and a key whose +// value wraps past its own line — in whichever spelling the file uses. All +// three used to be silent: the flow-style one reported success while the value +// stayed live, and the other two had their key line commented out while the +// rest of the value stayed behind, to be re-read as something else or to stop +// the file from parsing. The set direction already refuses what it cannot write +// correctly, so unset says so too. // -// This is a list of known shapes, not a decision procedure for the whole class: -// a value that spans lines in flow or quoted style lands its body in the same -// place and is still silent. A mapping-valued key is refused by the line -// matcher itself (mappingValueUnsetRefusal), which also covers single-segment -// keys this function never walks. +// The first two are named shapes with diagnoses of their own. The third is +// decided by where the entry ends rather than by the value's style +// (spansLinesUnsetRefusal), so the same value written on one line still +// comments out cleanly. A mapping-valued key is refused by the line matcher +// itself (mappingValueUnsetRefusal), which also covers single-segment keys this +// function never walks. An edit none of these names is still judged by its +// effect once the line matcher has made it (collateralUnsetRefusal). // -// Only a key that is actually present can be refused: unsetting a key that was -// never there has always been a successful no-op, and staying silent about a -// shape nobody asked to touch is the point. +// This function refuses only a key that is actually present: unsetting a key +// that was never there has always succeeded, and staying silent about a shape +// nobody asked to touch is the point. func unsupportedUnsetShape(content, key string) error { // Single-segment keys are checked too. The database-backed unset clears // the config.yaml layer for every key, so `notes: |` with an indented body @@ -1446,16 +1468,15 @@ func unsupportedUnsetShape(content, key string) error { // The flat spelling first, and by the same order the line matcher uses: it // comments a literal `sync.remote:` line out without ever walking the nested // path, so the body left behind orphans at the TOP level, where it stops the - // whole document from parsing rather than just its own section. - if top := root.Content[0]; top.Kind == yaml.MappingNode { - if idx := findMappingChild(top, key); idx != -1 { - if err := blockScalarUnsetRefusal(top.Content[idx+1], key); err != nil { - return err - } - } + // whole document from parsing rather than just its own section. A + // single-segment key is its own flat spelling, and the walk below would only + // find the same entry again. + if err := flatKeyUnsetRefusal(content, root.Content[0], key); err != nil || len(segments) == 1 { + return err } node := root.Content[0] + var keyNode *yaml.Node parents := make([]*yaml.Node, 0, len(segments)) for _, segment := range segments { if node.Kind != yaml.MappingNode { @@ -1466,20 +1487,58 @@ func unsupportedUnsetShape(content, key string) error { return nil } parents = append(parents, node) - node = node.Content[idx+1] + keyNode, node = node.Content[idx], node.Content[idx+1] } for i, parent := range parents { - if parent.Style&yaml.FlowStyle == 0 { - continue + if parent.Style&yaml.FlowStyle != 0 { + return flowStyleUnsetRefusal(key, segments[:i]) } - where := "the top level of this config file" - if i > 0 { - where = fmt.Sprintf("%q", strings.Join(segments[:i], ".")) - } - return fmt.Errorf("cannot unset %q: %s is written in flow style ({...}), which bd cannot edit; remove the key by hand", key, where) } - return blockScalarUnsetRefusal(node, key) + if err := blockScalarUnsetRefusal(node, key); err != nil { + return err + } + return spansLinesUnsetRefusal(content, keyNode, node, key) +} + +// flatKeyUnsetRefusal runs unsupportedUnsetShape's checks on the flat spelling +// of key: a top-level entry whose name is the whole key, dots included. +// +// Under a flow-style top level that entry is a key inside a flow-style mapping +// like any other, and the line matcher cannot edit it either. On the opening +// line (`{sync.remote: a, node_id: b}`) it is never matched at all, so the +// value stays live while the unset reports success. On a line of its own it +// can be matched, but that line may close the braces as well, or hold only the +// start of a value that wraps. So it gets the flow-style refusal, as a +// single-segment key there always has. The span probe would not hold here +// anyway: cut inside the mapping's own braces, the document fails to parse +// whatever the value is. +func flatKeyUnsetRefusal(content string, top *yaml.Node, key string) error { + if top.Kind != yaml.MappingNode { + return nil + } + idx := findMappingChild(top, key) + if idx == -1 { + return nil + } + if top.Style&yaml.FlowStyle != 0 { + return flowStyleUnsetRefusal(key, nil) + } + if err := blockScalarUnsetRefusal(top.Content[idx+1], key); err != nil { + return err + } + return spansLinesUnsetRefusal(content, top.Content[idx], top.Content[idx+1], key) +} + +// flowStyleUnsetRefusal refuses a key inside a flow-style mapping, which the +// line matcher cannot edit. parent is the path of that mapping, empty for the +// top level. +func flowStyleUnsetRefusal(key string, parent []string) error { + where := "the top level of this config file" + if len(parent) > 0 { + where = fmt.Sprintf("%q", strings.Join(parent, ".")) + } + return fmt.Errorf("cannot unset %q: %s is written in flow style ({...}), which bd cannot edit; remove the key by hand", key, where) } // blockScalarUnsetRefusal refuses a key whose value is written as a block @@ -1493,6 +1552,298 @@ func blockScalarUnsetRefusal(valueNode *yaml.Node, key string) error { return fmt.Errorf("cannot unset %q: its value is a block scalar (| or >), and commenting the key out would leave the body behind as a value of its own; remove the key by hand", key) } +// spansLinesUnsetRefusal refuses a key of a block mapping whose value starts on +// the key's own line and does not end there: a flow collection, a quoted string +// or a plain scalar wrapped onto the next line (`remote: {url: a,` then +// `branch: b}`). The line matcher comments out only the key's line, so the rest +// of the value would stay behind with its opening bracket or quote gone: as a +// line that re-reads as a key nobody set (`branch: b}`) or as the value of the +// enclosing key (`sync: b`), or one that stops the file from parsing. +// +// It is decided by where the entry ends, not by the value's style: the same +// collection written on one line comments out cleanly. Everything above an +// entry of a block mapping is either a finished entry or an enclosing block +// mapping, and those end with the document, so the document cut off after the +// key's line parses whenever the entry ends on that line, and fails while a +// flow collection or quoted string opened on it is still open. The cut counts +// lines the way yaml.v3 numbers them (yamlLineEnd), not by "\n" alone: a +// carriage return or a Unicode line separator above the key would otherwise +// move the cut to another line, refusing a value that ends on its own line +// and missing one that wraps. Cutting the document rather than parsing the one +// line on its own, as replaceFlatKeyLine does, keeps anchors in scope: +// `remote: *r` is a whole entry, but its alias resolves only against an anchor +// defined above it. +// +// A plain scalar continued onto a deeper line (`remote: file:///a`, then `b`) +// needs one more look, because it parses when cut after any of its lines, as a +// shorter value. The cut holds the key's line and nothing after it, so the +// key's value is the last node in it, and a value that started on the key's +// line and goes on reads shorter there than in the whole file. An alias or a +// collection is not compared: a cut that parses has already closed it. +func spansLinesUnsetRefusal(content string, keyNode, valueNode *yaml.Node, key string) error { + end, ok := yamlLineEnd(content, keyNode.Line) + if !ok { + // Nothing follows the key's line for its value to wrap onto. + return nil + } + var cut yaml.Node + if yaml.Unmarshal([]byte(content[:end]), &cut) == nil && !plainScalarGoesOn(&cut, keyNode, valueNode) { + return nil + } + return fmt.Errorf("cannot unset %q: its value spans more than one line, and commenting the key out would leave the rest of it behind; remove the key by hand", key) +} + +// plainScalarGoesOn reports whether a scalar value that starts on the key's +// line reads differently in cut, the document cut off after that line, than +// in the whole file: a plain scalar continued onto the lines below. +func plainScalarGoesOn(cut, keyNode, valueNode *yaml.Node) bool { + if valueNode.Kind != yaml.ScalarNode || valueNode.Line != keyNode.Line { + return false + } + last := cut + for len(last.Content) > 0 { + last = last.Content[len(last.Content)-1] + } + return last.Value != valueNode.Value +} + +// yamlLineEnd reports the offset of the line break that ends the given 1-based +// line of content, counting breaks as yaml.v3 does when it numbers node lines: +// LF, CR, CRLF as one break, NEL (U+0085) and the Unicode line and paragraph +// separators (U+2028, U+2029). ok is false when no break ends that line, +// because it is the last line or out of range. +func yamlLineEnd(content string, line int) (end int, ok bool) { + breaks := 0 + for i, r := range content { + switch { + case r == '\n' && i > 0 && content[i-1] == '\r': + continue // the LF of a CRLF, already counted at its CR + case r == '\n', r == '\r', r == '\u0085', r == '\u2028', r == '\u2029': + breaks++ + if breaks == line { + return i, true + } + } + } + return 0, false +} + +// collateralUnsetRefusal refuses an unset whose edit changed more of the file +// than the key. Both unset callers run it on commentOutYamlKey's result before +// they write. The refusals before the edit name the shapes they know, judged +// from the first document's parse; this one judges the edit by its effect, so +// it also catches the shapes they cannot see: a later duplicate of the key +// whose value wraps, the key in a later document, an anchor on the key's line +// that an alias further down still uses, or a line inside another value that +// only looks like the key, beside the key itself. A file that does not hold +// the key at all never gets here: commentOutYamlKey leaves it as it is. +// +// Read back, after must hold what before held, document by document, less +// some of the key's own entries in either spelling: nothing added, and nothing +// else removed or changed. Documents are compared by position because bd reads +// only the first one: commenting out all of a first document that no `---` +// opens would move the second one into its place. +// +// A later document that does not parse leaves the ones before it readable, the +// first of them being what bd reads, so those are compared all the same, and +// the edit must leave every one of them parsing. The document that does not +// parse, and any after it, has no reading to compare, before the edit or after +// it. A file whose first document does not parse is left to the line matcher's +// best effort, as it always has been. A file that nests deeper than +// maxYamlLeafDepth is refused: it costs too much to compare. +func collateralUnsetRefusal(before, after, key string) error { + was, wasErr := yamlDocuments(before) + if wasErr != nil && len(was) == 0 { + return nil + } + is, isErr := yamlDocuments(after) + if wasErr != nil && len(is) >= len(was) { + is, isErr = is[:len(was)], nil + } + wasLeaves, ok := yamlLeaves(was) + if !ok { + return fmt.Errorf("cannot unset %q: this file nests more than %d levels deep, too deep for bd to check what else commenting out the key would change; remove the key by hand", key, maxYamlLeafDepth) + } + isLeaves, ok := yamlLeaves(is) + if ok && isErr == nil && onlyKeyLeavesRemoved(wasLeaves, isLeaves, key) { + return nil + } + return fmt.Errorf("cannot unset %q: commenting out the lines that match it would change more of this file than the key itself; remove the key by hand", key) +} + +// yamlLeaf is one scalar of a YAML stream: the document it is in, the keys and +// list indexes that lead to it, and its value. An alias is a leaf of its own, +// naming the anchor it resolves to rather than expanding it, so the list never +// grows past the size of the file, however the aliases nest. +type yamlLeaf struct { + doc int + path []string + value string +} + +func (l yamlLeaf) id() string { + return fmt.Sprintf("%d %q %q", l.doc, l.path, l.value) +} + +// maxYamlLeafDepth is how many keys and list items deep yamlLeaves reads. A +// leaf carries the whole path to it, so listing a document's leaves costs +// their number times their depth, and yaml.v3 lets a document nest 10000 +// deep: one unset of a 45 KB config.yaml nested 9000 deep allocated 1.7 GB. +// No config bd reads comes near this depth. +const maxYamlLeafDepth = 64 + +// yamlLeaves lists the leaves of docs. A null scalar or an empty collection +// holds no leaf, so a section left holding nothing reads the same as one that +// is gone. ok is false, and no leaves come back, when a document nests more +// than maxYamlLeafDepth keys and list items deep. +func yamlLeaves(docs []*yaml.Node) (leaves []yamlLeaf, ok bool) { + for doc, root := range docs { + if yamlNestsDeeper(root, maxYamlLeafDepth) { + return nil, false + } + leaves = appendYamlLeaves(leaves, doc, nil, root) + } + return leaves, true +} + +// yamlNestsDeeper reports whether some node under n has more than depth +// mappings and sequences above it, counting n. Under a document, those are the +// keys and list items on the node's path. It looks no deeper than that. +func yamlNestsDeeper(n *yaml.Node, depth int) bool { + if n.Kind == yaml.MappingNode || n.Kind == yaml.SequenceNode { + depth-- + } + for _, child := range n.Content { + if depth < 0 || yamlNestsDeeper(child, depth) { + return true + } + } + return false +} + +// yamlDocuments decodes the documents of content, in order. When one does not +// parse, the documents decoded before it come back with the error. +func yamlDocuments(content string) ([]*yaml.Node, error) { + dec := yaml.NewDecoder(strings.NewReader(content)) + var docs []*yaml.Node + for { + var root yaml.Node + err := dec.Decode(&root) + if errors.Is(err, io.EOF) { + return docs, nil + } + if err != nil { + return docs, err + } + docs = append(docs, &root) + } +} + +// appendYamlLeaves appends the leaves of n, which path leads to, to leaves. +func appendYamlLeaves(leaves []yamlLeaf, doc int, path []string, n *yaml.Node) []yamlLeaf { + switch n.Kind { + case yaml.DocumentNode: + for _, child := range n.Content { + leaves = appendYamlLeaves(leaves, doc, path, child) + } + case yaml.MappingNode: + for i := 0; i+1 < len(n.Content); i += 2 { + leaves = appendYamlLeaves(leaves, doc, append(path[:len(path):len(path)], n.Content[i].Value), n.Content[i+1]) + } + case yaml.SequenceNode: + for i, item := range n.Content { + leaves = appendYamlLeaves(leaves, doc, append(path[:len(path):len(path)], strconv.Itoa(i)), item) + } + case yaml.AliasNode: + // By where its anchor is, not by name alone: YAML lets a later anchor + // reuse a name, and with the later one's line commented out the alias + // would resolve to the earlier one, reading the same while its value + // changed. Commenting a line out moves no other node, so an alias + // whose anchor is untouched reads the same. + leaves = append(leaves, yamlLeaf{doc: doc, path: path, value: fmt.Sprintf("*%s@%d:%d", n.Value, n.Alias.Line, n.Alias.Column)}) + case yaml.ScalarNode: + if n.ShortTag() != "!!null" { + leaves = append(leaves, yamlLeaf{doc: doc, path: path, value: n.Value}) + } + } + return leaves +} + +// onlyKeyLeavesRemoved reports whether is holds exactly was's leaves less some +// that belong to key: under the top-level entry named key, dots and all, or +// under its nested path, the two spellings the line matcher comments out. +func onlyKeyLeavesRemoved(was, is []yamlLeaf, key string) bool { + left := make(map[string]int, len(was)) + for _, leaf := range was { + left[leaf.id()]++ + } + for _, leaf := range is { + id := leaf.id() + if left[id] == 0 { + return false + } + left[id]-- + } + segments := strings.Split(key, ".") + for _, leaf := range was { + p := leaf.path + ofKey := (len(p) > 0 && p[0] == key) || (len(p) >= len(segments) && slices.Equal(p[:len(segments)], segments)) + if left[leaf.id()] > 0 && !ofKey { + return false + } + } + return true +} + +// yamlHoldsKey reports whether some document of content has an entry for key +// in either spelling the line matcher comments out: a top-level entry named +// key, dots and all, or its nested path. An entry whose value is null holds +// the key too, and so does one under either copy of a section the file writes +// twice, since the matcher comments out both. +// +// When a later document does not parse, the documents before it answer: bd +// reads the first of them, and an edit to them is checked like any other +// (collateralUnsetRefusal). The document that does not parse, and any after +// it, is read by nobody, so a key there is not set. A file whose first +// document does not parse gives no answer, so it is taken to hold the key, +// and the line matcher keeps its best effort there, as it always has. +func yamlHoldsKey(content, key string) bool { + docs, err := yamlDocuments(content) + if err != nil && len(docs) == 0 { + return true + } + segments := strings.Split(key, ".") + for _, doc := range docs { + for _, top := range doc.Content { + if mappingHoldsPath(top, []string{key}) || mappingHoldsPath(top, segments) { + return true + } + } + } + return false +} + +// mappingHoldsPath reports whether n leads down path through mappings, one key +// per step. Every entry of a name that repeats is followed. +func mappingHoldsPath(n *yaml.Node, path []string) bool { + level := []*yaml.Node{n} + for _, name := range path { + var next []*yaml.Node + for _, m := range level { + if m.Kind != yaml.MappingNode { + continue + } + for i := 0; i+1 < len(m.Content); i += 2 { + if k := m.Content[i]; k.Kind == yaml.ScalarNode && k.Value == name { + next = append(next, m.Content[i+1]) + } + } + } + level = next + } + return len(level) > 0 +} + // blockScalarIndent reports the indentation of a key whose value is a literal // or folded block scalar (`notes: |`), or -1 when the line opens no block. func blockScalarIndent(line string) int { diff --git a/internal/config/yaml_nested_roundtrip_test.go b/internal/config/yaml_nested_roundtrip_test.go index df464551a..91ba0b5b2 100644 --- a/internal/config/yaml_nested_roundtrip_test.go +++ b/internal/config/yaml_nested_roundtrip_test.go @@ -410,6 +410,110 @@ func TestUnsetRemovesADottedKeyInEveryShape(t *testing.T) { key: "dolt.host", survivors: map[string]string{"other.dolt.host": "should-survive"}, }, + { + // A flow collection that opens and closes on the key's own line is + // a whole entry on that line, so commenting the line out removes + // it cleanly. TestUnsetRefusesShapesItCannotEdit refuses the same + // collection wrapped onto the next line; that refusal is decided by + // where the entry ends, not by its style, and must not reach this. + name: "flow mapping leaf on one line", + seed: "sync:\n remote: {url: a, branch: b}\nnode_id: mini\n", + key: "sync.remote", + survivors: map[string]string{"node_id": "mini"}, + }, + { + // An alias resolves only against an anchor defined above it, so + // the key's line parsed on its own fails on `*r` even though the + // entry ends right there. The refusal must judge the line with the + // anchor in scope, or it refuses a value it can remove cleanly. + name: "alias to an anchor above", + seed: "sync:\n remote: &r file:///a\nbackup:\n remote: *r\n", + key: "backup.remote", + survivors: map[string]string{"sync.remote": "file:///a"}, + }, + { + // The same holds for an alias inside a one-line collection, which an + // exception for a value that is a bare alias would not cover. + name: "alias inside a one-line collection", + seed: "sync:\n remote: &r file:///a\nbackup:\n remotes: [*r, file:///b]\n", + key: "backup.remotes", + survivors: map[string]string{"sync.remote": "file:///a"}, + }, + { + // yaml.v3 ends a line at a Unicode line separator or a lone carriage + // return as well as at LF, even inside a quoted string, so either + // one above the key numbers its line one past the line a "\n" split + // finds it on. Cut by "\n", the probe ended inside the collection + // below and refused a value that ends on its own line. + name: "one-line value below a line separator", + seed: "notes: \"a\u2028b\"\nremote: x\nother: {a: 1,\n b: 2}\n", + key: "remote", + survivors: map[string]string{"other.b": "2"}, + }, + { + name: "one-line value below a lone carriage return", + seed: "notes: \"a\rb\"\nremote: x\nother: {a: 1,\n b: 2}\n", + key: "remote", + survivors: map[string]string{"other.b": "2"}, + }, + { + // With no newline after it, the key's line is the last one, which a + // "\n" split counts one short of yaml.v3: nothing follows it for the + // value to wrap onto. + name: "last line with no newline, below a line separator", + seed: "notes: \"a\u2028b\"\nremote: x", + key: "remote", + survivors: map[string]string{"notes": "a\u2028b"}, + }, + { + // A flat key under a flow-style top level is refused, but only when + // it is there: an unset with nothing to remove still succeeds. + name: "absent from a flow-style top level", + seed: "{node_id: mini}\n", + key: "sync.remote", + survivors: map[string]string{"node_id": "mini"}, + }, + { + // A plain scalar that ends on its line reads the same cut after it + // as in the whole file, and a comment below it is not a + // continuation, however deep. + name: "plain scalar with a deeper comment below it", + seed: "sync:\n remote: file:///a\n # the shared remote\n branch: b\n", + key: "sync.remote", + survivors: map[string]string{"sync.branch": "b"}, + }, + { + // Later documents are checked as well, but an edit to the first one + // leaves them as they were. + name: "key in the first of two documents", + seed: "sync:\n remote: x\nnode_id: mini\n---\nother: y\n", + key: "sync.remote", + survivors: map[string]string{"node_id": "mini"}, + }, + { + // Opened by its own ---, the first document stays a document, if an + // empty one, after its only key is commented out, so the second + // document does not move up into its place. + name: "only key of a first document that --- opens", + seed: "---\nremote: x\n---\nnode_id: mini\n", + key: "remote", + }, + { + // A later document that does not parse leaves the first one + // readable, and the edit there is checked and made all the same. + name: "key above a document that does not parse", + seed: "sync:\n remote: x\nnode_id: mini\n---\nother: [unclosed\n", + key: "sync.remote", + survivors: map[string]string{"node_id": "mini"}, + }, + { + // The check of the edit reads 64 keys and list items deep, which is + // as deep as this sibling goes. + name: "beside a value nested 64 deep", + seed: "sync:\n remote: x\nnode_id: mini\ndeep: " + strings.Repeat("[", 63) + "x" + strings.Repeat("]", 63) + "\n", + key: "sync.remote", + survivors: map[string]string{"node_id": "mini"}, + }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -475,6 +579,21 @@ func TestUnsetRefusesShapesItCannotEdit(t *testing.T) { }{ {name: "flow style section", seed: "sync: {remote: \"file:///origin.git\"}\n", key: "sync.remote", want: "flow style"}, {name: "flow style nested deeper", seed: "dolt:\n limits: {host: mini}\n", key: "dolt.limits.host", want: "flow style"}, + // A flow-style top level spread over lines is still named for its + // style. The key's own value ends on its line; only the mapping + // around it goes on. + {name: "flow style top level spread over lines", seed: "{notes: x,\n node_id: mini}\n", key: "notes", want: "flow style"}, + // A flat key is a key of the top level, so a flow-style top level holds + // it in a flow-style mapping too. The line matcher never finds it on + // the opening line, takes the closing brace with it, or leaves the rest + // of a wrapped value behind, so it gets the same refusal, even for a + // value on a line of its own. + {name: "flat key on the opening line of a flow-style top level", seed: "{sync.remote: a, node_id: b}\n", key: "sync.remote", want: "the top level of this config file is written in flow style"}, + {name: "flat key wrapped from the opening line of a flow-style top level", seed: "{sync.remote: {url: a,\n branch: b}}\n", key: "sync.remote", want: "the top level of this config file is written in flow style"}, + {name: "flat key whose flow value wraps in a flow-style top level", seed: "{\n sync.remote: {url: a,\n branch: b},\n node_id: mini\n}\n", key: "sync.remote", want: "the top level of this config file is written in flow style"}, + {name: "flat key whose quoted value wraps in a flow-style top level", seed: "{\n sync.remote: \"file:///a\n /b\",\n node_id: mini\n}\n", key: "sync.remote", want: "the top level of this config file is written in flow style"}, + {name: "flat key on the line that closes a flow-style top level", seed: "{\n node_id: mini,\n sync.remote: x}\n", key: "sync.remote", want: "the top level of this config file is written in flow style"}, + {name: "flat key on a line of its own in a flow-style top level", seed: "{\n sync.remote: file:///a,\n node_id: mini\n}\n", key: "sync.remote", want: "the top level of this config file is written in flow style"}, {name: "literal block scalar", seed: "sync:\n remote: |\n line1\n line2\n", key: "sync.remote", want: "block scalar"}, {name: "folded block scalar", seed: "sync:\n remote: >\n line1\n", key: "sync.remote", want: "block scalar"}, // The flat spelling reaches the same body-left-behind outcome by the @@ -507,6 +626,84 @@ func TestUnsetRefusesShapesItCannotEdit(t *testing.T) { {name: "plain scalar on the next line", seed: "dolt.mode:\n server\n", key: "dolt.mode", want: "lines beneath it"}, // A CRLF blank line is a lone carriage return, not content at column 0. {name: "mapping behind a CRLF blank line", seed: "backup:\r\n\r\n enabled: false\r\n", key: "backup", want: "lines beneath it"}, + // A value that starts on the key's line and wraps onto the next one: + // commenting the key line out leaves the rest of the value behind + // without its opening bracket or quote, as a line that re-reads as a + // key of the section (`sync.branch: "b}"`) or one that stops the file + // from parsing. The same value written on one line still comments out + // cleanly. + {name: "flow mapping wrapped onto the next line", seed: "sync:\n remote: {url: a,\n branch: b}\nnode_id: mini\n", key: "sync.remote", want: "spans more than one line"}, + // Under the flat spelling the rest lands at the top level, where the + // indented `branch: b}` parses as the whole document and the keys + // after it (`node_id`) stop reading at all. + {name: "flow mapping wrapped under the flat spelling", seed: "sync.remote: {url: a,\n branch: b}\nnode_id: mini\n", key: "sync.remote", want: "spans more than one line"}, + // The rest can sit at the key's own indentation, where it reads as a + // sibling key rather than as lines beneath this one, so no indentation + // rule sees it; only where the entry ends does. + {name: "flow mapping that closes at the key's own indentation", seed: "sync:\n remote: {url: a,\n branch: b}\nnode_id: mini\n", key: "sync.remote", want: "spans more than one line"}, + {name: "flow sequence wrapped onto the next line", seed: "types.custom: [step,\n slot]\nnode_id: mini\n", key: "types.custom", want: "spans more than one line"}, + // A quoted string can wrap as well; YAML folds the break into a space. + // Left behind, `line two"` stops the file from parsing. + {name: "double-quoted string wrapped onto the next line", seed: "notes: \"line one\n line two\"\nnode_id: mini\n", key: "notes", want: "spans more than one line"}, + {name: "single-quoted string wrapped under the flat spelling", seed: "dolt.host: '10.0.0.1\n more'\nnode_id: mini\n", key: "dolt.host", want: "spans more than one line"}, + // The probe reads the file after CRLF is normalized to LF, and a + // refusal writes nothing, so a CRLF file keeps its line endings too. + {name: "flow mapping wrapped in a CRLF file", seed: "sync:\r\n remote: {url: a,\r\n branch: b}\r\nnode_id: mini\r\n", key: "sync.remote", want: "spans more than one line"}, + // yaml.v3 ends a line at a Unicode line separator or a lone carriage + // return too, even inside a quoted string. Above the key, either one + // numbers the key's line one past the line a "\n" split finds it on, + // so a probe cut by "\n" took in the whole wrapped value and let it + // through. + {name: "wrapped value below a line separator", seed: "notes: \"a\u2028b\"\nsync:\n remote: {url: a,\n branch: b}\nnode_id: mini\n", key: "sync.remote", want: "spans more than one line"}, + {name: "wrapped value below a lone carriage return", seed: "notes: \"a\rb\"\nsync:\n remote: {url: a,\n branch: b}\nnode_id: mini\n", key: "sync.remote", want: "spans more than one line"}, + // In the key's own value, the separator is a wrap: the commented-out + // line ends at it, and `b"` is left behind to stop the file from + // parsing. + {name: "quoted value holding a line separator", seed: "remote: \"a\u2028b\"\nnode_id: mini\n", key: "remote", want: "spans more than one line"}, + // A plain scalar continued onto a deeper line wraps too. Cut after the + // key's line it still parses, as a shorter value, so the probe compares + // the two. Left behind, the rest re-reads as the value of the enclosing + // key (`sync: /b`), or stops the file from parsing. + {name: "plain scalar continued onto a deeper line", seed: "sync:\n remote: file:///a\n /b\nnode_id: mini\n", key: "sync.remote", want: "spans more than one line"}, + {name: "plain scalar continued past a blank line", seed: "sync:\n remote: file:///a\n\n /b\nnode_id: mini\n", key: "sync.remote", want: "spans more than one line"}, + {name: "plain scalar continued under the flat spelling", seed: "dolt.host: 10.0.0.1\n more\nnode_id: mini\n", key: "dolt.host", want: "spans more than one line"}, + {name: "single-segment plain scalar continued", seed: "notes: a long note that the\n editor wrapped\nnode_id: mini\n", key: "notes", want: "spans more than one line"}, + // The refusals above judge the first document's parse before anything + // is edited. What gets past them is refused by what the edit would do: + // the file read back has to say what it said before, less the key. + // The probe sees the first of two duplicates, and the line matcher + // comments out both, leaving `line two"` behind. + {name: "later duplicate whose value wraps", seed: "notes: ok\nnotes: \"line one\n line two\"\nnode_id: mini\n", key: "notes", want: "more of this file than the key"}, + // A section the file writes twice holds the key in its second copy, + // which the refusals before the edit, reading the first, never see. + {name: "wrapped value under the second copy of its section", seed: "sync:\n branch: b\nsync:\n remote: \"x\n y\"\nnode_id: mini\n", key: "sync.remote", want: "more of this file than the key"}, + // A later document is not in the first document's parse at all. + {name: "wrapped value in a later document", seed: "node_id: mini\n---\nnotes: \"line one\n line two\"\n", key: "notes", want: "more of this file than the key"}, + // The anchor goes with the key's line, and the alias below it then + // names nothing, so the file stops parsing. + {name: "anchor on the key's line used further down", seed: "sync:\n remote: &r file:///a\nbackup:\n remote: *r\n", key: "sync.remote", want: "more of this file than the key"}, + // YAML lets a later anchor reuse a name. With the key's line commented + // out, the alias below resolves to the anchor above it instead, so the + // file still parses, but backup.remote reads keep rather than a. + {name: "anchor on the key's line reusing the name of one above", seed: "x: &r keep\nsync:\n remote: &r a\nbackup:\n remote: *r\n", key: "sync.remote", want: "more of this file than the key"}, + // A line that only looks like the key is someone else's even when the + // key is set too: the line matcher comments out both lines, and notes + // would change. TestUnsetOfAnAbsentKeyStillSucceeds has the same line + // in a file that does not hold the key. + {name: "the key beside a line inside another value that looks like it", seed: "notes: \"see\njira.url: https://x\"\njira.url: https://y\n", key: "jira.url", want: "more of this file than the key"}, + // bd reads only the first document. With its only key commented out, + // that document is gone, and node_id would be read in its place. + {name: "only key of the first document", seed: "remote: x\n---\nnode_id: mini\n", key: "remote", want: "more of this file than the key"}, + // A later document that does not parse leaves the ones before it + // readable, so an edit to them is checked as in any other file. Here + // notes would change, and with the first document gone, the one that + // does not parse would be read in its place. + {name: "the key beside a line that looks like it, above a document that does not parse", seed: "notes: \"see\njira.url: https://x\"\njira.url: https://y\n---\nother: [unclosed\n", key: "jira.url", want: "more of this file than the key"}, + {name: "only key of the first document, above one that does not parse", seed: "remote: x\n---\nother: [unclosed\n", key: "remote", want: "more of this file than the key"}, + // Each leaf the check compares carries its whole path, so a file + // nested past 64 keys and list items deep costs too much to compare, + // and yaml.v3 lets one nest 10000 deep. + {name: "beside a value nested 65 deep", seed: "sync:\n remote: x\ndeep: " + strings.Repeat("[", 64) + "x" + strings.Repeat("]", 64) + "\n", key: "sync.remote", want: "nests more than 64 levels deep"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -539,11 +736,70 @@ func TestUnsetRefusesShapesItCannotEdit(t *testing.T) { // A shape bd cannot edit is only worth refusing when the key is actually there. // Unsetting a key that was never set has always been a successful no-op, and a -// flow-style mapping elsewhere in the file is not the operator's problem. +// flow-style mapping elsewhere in the file is not the operator's problem. Nor +// is a line that only looks like the key: the line matcher reads lines, and +// commenting such a line out edits a value nobody named, while refusing it +// fails an unset with nothing to remove. Either way the file stays as it was. func TestUnsetOfAnAbsentKeyStillSucceeds(t *testing.T) { + cases := []struct { + name string + seed string + key string + }{ + {name: "flow-style mapping elsewhere", seed: "sync: {branch: beads-sync}\ndolt:\n port: 3307\n", key: "sync.remote"}, + // The matching line is the second line of notes, in either spelling. + {name: "line inside another value that looks like the key", seed: "sync:\n notes: \"x\n remote: y\"\n", key: "sync.remote"}, + {name: "line inside another value that looks like the flat key", seed: "notes: \"see\njira.url: https://x\"\n", key: "jira.url"}, + // sync is a list, so sync.remote is not set, but the nested walk + // matches the key of its item. An alias there is a value like any + // other, though it is not expanded. The matcher refused the same key + // with a mapping beneath it as a value that goes on below, and + // commented it out with no value at all, which left nothing to compare. + {name: "key inside a list item", seed: "base: &r file:///a\nsync:\n -\n remote: *r\n", key: "sync.remote"}, + {name: "key with a mapping beneath it inside a list item", seed: "sync:\n -\n remote:\n url: a\n", key: "sync.remote"}, + {name: "key with no value inside a list item", seed: "sync:\n -\n remote:\n", key: "sync.remote"}, + // A later document that does not parse leaves the first one readable, + // and that one does not hold the key either. + {name: "line that looks like the key, above a document that does not parse", seed: "notes: \"see\njira.url: https://x\"\n---\nother: [unclosed\n", key: "jira.url"}, + // Nobody reads a document that does not parse, so a key there is not + // set. + {name: "key only in a document that does not parse", seed: "node_id: mini\n---\nsync:\n remote: x\nother: [unclosed\n", key: "sync.remote"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.yaml") + if err := os.WriteFile(path, []byte(tc.seed), 0o600); err != nil { + t.Fatalf("seed: %v", err) + } + + t.Setenv("BEADS_DIR", dir) + changed, err := UnsetYamlConfig(tc.key) + if err != nil { + t.Fatalf("UnsetYamlConfig on an unset key: %v", err) + } + if changed { + t.Error("UnsetYamlConfig on an unset key reported a write") + } + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read: %v", err) + } + if string(body) != tc.seed { + t.Errorf("the no-op unset changed the file:\n%s", body) + } + }) + } +} + +// A file whose first document yaml.v3 cannot parse has no reading to check an +// edit against, so the line matcher's best effort stands there, as it always +// has: the key's line is commented out and the rest of the file is left as it +// was. +func TestUnsetInAFileThatDoesNotParseStillCommentsTheKeyOut(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "config.yaml") - seed := "sync: {branch: beads-sync}\ndolt:\n port: 3307\n" + seed := "sync:\n remote: x\nother: [unclosed\n" if err := os.WriteFile(path, []byte(seed), 0o600); err != nil { t.Fatalf("seed: %v", err) } @@ -551,17 +807,17 @@ func TestUnsetOfAnAbsentKeyStillSucceeds(t *testing.T) { t.Setenv("BEADS_DIR", dir) changed, err := UnsetYamlConfig("sync.remote") if err != nil { - t.Fatalf("UnsetYamlConfig on an unset key: %v", err) + t.Fatalf("UnsetYamlConfig in a file that does not parse: %v", err) } - if changed { - t.Error("UnsetYamlConfig on an unset key reported a write") + if !changed { + t.Error("UnsetYamlConfig reported no write for a key it commented out") } body, err := os.ReadFile(path) if err != nil { t.Fatalf("read: %v", err) } - if got := GetStringFromDir(dir, "sync.branch"); got != "beads-sync" { - t.Errorf("the no-op unset changed sync.branch to %q:\n%s", got, body) + if want := "sync:\n # remote: x\nother: [unclosed\n"; string(body) != want { + t.Errorf("the unset left\n%q\nwant\n%q", body, want) } } diff --git a/internal/config/yaml_unset_newline_test.go b/internal/config/yaml_unset_newline_test.go index 47dea8907..09ac65195 100644 --- a/internal/config/yaml_unset_newline_test.go +++ b/internal/config/yaml_unset_newline_test.go @@ -1,10 +1,13 @@ package config import ( + "fmt" "os" "path/filepath" "strings" "testing" + + "gopkg.in/yaml.v3" ) // Neither an unset nor a set may also rewrite the end of the file. @@ -255,6 +258,121 @@ func TestUnsetOfAPresentKeyInACRLFFileReportsTheWrite(t *testing.T) { }) } +// The user-global config.yaml goes through the same line matcher, so a value +// that wraps onto the next line is refused there as well, with nothing written +// and no change reported. node_id has a single segment, so this is the +// top-level half of the refusal. +func TestUnsetUserYamlConfigRefusesAValueThatSpansLines(t *testing.T) { + const body = "node_id: \"mini\n box\"\n" + + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + path, err := UserConfigYamlPath() + if err != nil { + t.Fatalf("UserConfigYamlPath: %v", err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + + changed, err := UnsetUserYamlConfig("node_id") + if err == nil { + after, _ := os.ReadFile(path) //nolint:gosec,errcheck // test-owned temp path; diagnostic + t.Fatalf("UnsetUserYamlConfig reported success on a value that spans two lines:\n%q", string(after)) + } + if !strings.Contains(err.Error(), "node_id") || !strings.Contains(err.Error(), "spans more than one line") { + t.Errorf("the error names neither the key nor the shape: %v", err) + } + if changed { + t.Error("UnsetUserYamlConfig() changed = true for an unset it refused") + } + after, err := os.ReadFile(path) //nolint:gosec // test-owned temp path + if err != nil { + t.Fatal(err) + } + if string(after) != body { + t.Errorf("a refused unset still changed the file:\ngot: %q\nwant: %q", string(after), body) + } +} + +// The user-global unset checks the edit by its effect too, before it writes. +// node_id is not in the first document, so no refusal before the edit looks +// at it, but the line matcher still finds it in the second, where commenting +// out its first line would leave `box"` behind. +func TestUnsetUserYamlConfigRefusesAnEditThatChangesMoreThanTheKey(t *testing.T) { + const body = "metrics:\n disabled: true\n---\nnode_id: \"mini\n box\"\n" + + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + path, err := UserConfigYamlPath() + if err != nil { + t.Fatalf("UserConfigYamlPath: %v", err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + + changed, err := UnsetUserYamlConfig("node_id") + if err == nil { + after, _ := os.ReadFile(path) //nolint:gosec,errcheck // test-owned temp path; diagnostic + t.Fatalf("UnsetUserYamlConfig reported success on an edit that breaks the second document:\n%q", string(after)) + } + if !strings.Contains(err.Error(), "node_id") || !strings.Contains(err.Error(), "more of this file than the key") { + t.Errorf("the error names neither the key nor the reason: %v", err) + } + if changed { + t.Error("UnsetUserYamlConfig() changed = true for an unset it refused") + } + after, err := os.ReadFile(path) //nolint:gosec // test-owned temp path + if err != nil { + t.Fatal(err) + } + if string(after) != body { + t.Errorf("a refused unset still changed the file:\ngot: %q\nwant: %q", string(after), body) + } +} + +// The user-global unset of a key the file does not hold is the same no-op as +// the project one, though a line inside another value looks like the key: +// (false, nil) and the file as it was, not a refusal to edit notes. +func TestUnsetUserYamlConfigOfAnAbsentKeyLeavesTheFileAlone(t *testing.T) { + const body = "metrics:\n disabled: true\nnotes: \"see\nnode_id: mini\"\n" + + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + path, err := UserConfigYamlPath() + if err != nil { + t.Fatalf("UserConfigYamlPath: %v", err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + + changed, err := UnsetUserYamlConfig("node_id") + if err != nil { + t.Fatalf("UnsetUserYamlConfig on a key the file does not hold: %v", err) + } + if changed { + t.Error("UnsetUserYamlConfig() changed = true for a key the file does not hold") + } + after, err := os.ReadFile(path) //nolint:gosec // test-owned temp path + if err != nil { + t.Fatal(err) + } + if string(after) != body { + t.Errorf("the no-op unset changed the file:\ngot: %q\nwant: %q", string(after), body) + } +} + // The set twin of TestCommentOutYamlKeyPreservesTheTrailingNewline. // updateYamlKey's flat-key path read the document with bufio.Scanner and joined // the lines back with "\n" — the shape commentOutYamlKey used to have — so it @@ -468,3 +586,51 @@ func TestUpdateYamlKeyLeavesTheNestedPathAlone(t *testing.T) { }) } } + +// The span refusal cuts the document after the key's line as yaml.v3 numbers +// it, and yaml.v3 ends a line at more than LF: at a CR, at a CRLF (once), at a +// NEL, and at the Unicode line and paragraph separators. Counting any of them +// differently moves the cut onto another line, which refuses a value that ends +// on its own line or misses one that wraps. An unset normalizes CRLF before the +// probe runs, so a CRLF reaches the count only as what is left of a CR CR LF; +// the count has to get it right all the same. Each break is also tried as the +// file's first byte, a blank first line, where the look back for a CR before +// an LF has nothing to look at. +func TestYamlLineEndCutsWhereYamlV3EndsALine(t *testing.T) { + lineBreaks := map[string]string{ + "LF": "\n", "CR": "\r", "CRLF": "\r\n", "CR CRLF": "\r\r\n", + "NEL": "\u0085", "LS": "\u2028", "PS": "\u2029", + } + for name, lineBreak := range lineBreaks { + for _, lead := range []string{"", lineBreak} { + t.Run(fmt.Sprintf("%s lead %q", name, lead), func(t *testing.T) { + testYamlLineEndCuts(t, lead+"a: 1"+lineBreak+"b: 2"+lineBreak+"c: 3") + }) + } + } + // Out of range, before line 1 or past the last line, there is no cut. + for _, line := range []int{0, 4} { + if end, ok := yamlLineEnd("a: 1\nb: 2\nc: 3\n", line); ok { + t.Errorf("yamlLineEnd(line %d of 3) = %d, true; want ok = false", line, end) + } + } +} + +func testYamlLineEndCuts(t *testing.T, content string) { + t.Helper() + var root yaml.Node + if err := yaml.Unmarshal([]byte(content), &root); err != nil { + t.Fatalf("parse %q: %v", content, err) + } + top := root.Content[0] + for i := 0; i < len(top.Content); i += 2 { + key, value := top.Content[i], top.Content[i+1] + end, ok := yamlLineEnd(content, key.Line) + if last := i == len(top.Content)-2; ok == last { + t.Fatalf("yamlLineEnd(%q, %d) ok = %v for key %s", content, key.Line, ok, key.Value) + } + if entry := key.Value + ": " + value.Value; ok && !strings.HasSuffix(content[:end], entry) { + t.Errorf("the cut after line %d (key %s) is %q, want it to end with %q", key.Line, key.Value, content[:end], entry) + } + } +}