encode.go 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957
  1. // Copyright 2020 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. package jsontext
  5. import (
  6. "bytes"
  7. "io"
  8. "math/bits"
  9. "github.com/go-json-experiment/json/internal/jsonflags"
  10. "github.com/go-json-experiment/json/internal/jsonopts"
  11. "github.com/go-json-experiment/json/internal/jsonwire"
  12. )
  13. // Encoder is a streaming encoder from raw JSON tokens and values.
  14. // It is used to write a stream of top-level JSON values,
  15. // each terminated with a newline character.
  16. //
  17. // [Encoder.WriteToken] and [Encoder.WriteValue] calls may be interleaved.
  18. // For example, the following JSON value:
  19. //
  20. // {"name":"value","array":[null,false,true,3.14159],"object":{"k":"v"}}
  21. //
  22. // can be composed with the following calls (ignoring errors for brevity):
  23. //
  24. // e.WriteToken(ObjectStart) // {
  25. // e.WriteToken(String("name")) // "name"
  26. // e.WriteToken(String("value")) // "value"
  27. // e.WriteValue(Value(`"array"`)) // "array"
  28. // e.WriteToken(ArrayStart) // [
  29. // e.WriteToken(Null) // null
  30. // e.WriteToken(False) // false
  31. // e.WriteValue(Value("true")) // true
  32. // e.WriteToken(Float(3.14159)) // 3.14159
  33. // e.WriteToken(ArrayEnd) // ]
  34. // e.WriteValue(Value(`"object"`)) // "object"
  35. // e.WriteValue(Value(`{"k":"v"}`)) // {"k":"v"}
  36. // e.WriteToken(ObjectEnd) // }
  37. //
  38. // The above is one of many possible sequence of calls and
  39. // may not represent the most sensible method to call for any given token/value.
  40. // For example, it is probably more common to call [Encoder.WriteToken] with a string
  41. // for object names.
  42. type Encoder struct {
  43. s encoderState
  44. }
  45. // encoderState is the low-level state of Encoder.
  46. // It has exported fields and method for use by the "json" package.
  47. type encoderState struct {
  48. state
  49. encodeBuffer
  50. jsonopts.Struct
  51. SeenPointers map[any]struct{} // only used when marshaling; identical to json.seenPointers
  52. }
  53. // encodeBuffer is a buffer split into 2 segments:
  54. //
  55. // - buf[0:len(buf)] // written (but unflushed) portion of the buffer
  56. // - buf[len(buf):cap(buf)] // unused portion of the buffer
  57. type encodeBuffer struct {
  58. Buf []byte // may alias wr if it is a bytes.Buffer
  59. // baseOffset is added to len(buf) to obtain the absolute offset
  60. // relative to the start of io.Writer stream.
  61. baseOffset int64
  62. wr io.Writer
  63. // maxValue is the approximate maximum Value size passed to WriteValue.
  64. maxValue int
  65. // unusedCache is the buffer returned by the UnusedBuffer method.
  66. unusedCache []byte
  67. // bufStats is statistics about buffer utilization.
  68. // It is only used with pooled encoders in pools.go.
  69. bufStats bufferStatistics
  70. }
  71. // NewEncoder constructs a new streaming encoder writing to w
  72. // configured with the provided options.
  73. // It flushes the internal buffer when the buffer is sufficiently full or
  74. // when a top-level value has been written.
  75. //
  76. // If w is a [bytes.Buffer], then the encoder appends directly into the buffer
  77. // without copying the contents from an intermediate buffer.
  78. func NewEncoder(w io.Writer, opts ...Options) *Encoder {
  79. e := new(Encoder)
  80. e.Reset(w, opts...)
  81. return e
  82. }
  83. // Reset resets an encoder such that it is writing afresh to w and
  84. // configured with the provided options. Reset must not be called on
  85. // a Encoder passed to the [encoding/json/v2.MarshalerTo.MarshalJSONTo] method
  86. // or the [encoding/json/v2.MarshalToFunc] function.
  87. func (e *Encoder) Reset(w io.Writer, opts ...Options) {
  88. switch {
  89. case e == nil:
  90. panic("jsontext: invalid nil Encoder")
  91. case w == nil:
  92. panic("jsontext: invalid nil io.Writer")
  93. case e.s.Flags.Get(jsonflags.WithinArshalCall):
  94. panic("jsontext: cannot reset Encoder passed to json.MarshalerTo")
  95. }
  96. e.s.reset(nil, w, opts...)
  97. }
  98. func (e *encoderState) reset(b []byte, w io.Writer, opts ...Options) {
  99. e.state.reset()
  100. e.encodeBuffer = encodeBuffer{Buf: b, wr: w, bufStats: e.bufStats}
  101. if bb, ok := w.(*bytes.Buffer); ok && bb != nil {
  102. e.Buf = bb.Bytes()[bb.Len():] // alias the unused buffer of bb
  103. }
  104. e.Struct = jsonopts.Struct{}
  105. e.Struct.Join(opts...)
  106. if e.Flags.Get(jsonflags.Multiline) {
  107. if !e.Flags.Has(jsonflags.SpaceAfterColon) {
  108. e.Flags.Set(jsonflags.SpaceAfterColon | 1)
  109. }
  110. if !e.Flags.Has(jsonflags.SpaceAfterComma) {
  111. e.Flags.Set(jsonflags.SpaceAfterComma | 0)
  112. }
  113. if !e.Flags.Has(jsonflags.Indent) {
  114. e.Flags.Set(jsonflags.Indent | 1)
  115. e.Indent = "\t"
  116. }
  117. }
  118. }
  119. // NeedFlush determines whether to flush at this point.
  120. func (e *encoderState) NeedFlush() bool {
  121. // NOTE: This function is carefully written to be inlinable.
  122. // Avoid flushing if e.wr is nil since there is no underlying writer.
  123. // Flush if less than 25% of the capacity remains.
  124. // Flushing at some constant fraction ensures that the buffer stops growing
  125. // so long as the largest Token or Value fits within that unused capacity.
  126. return e.wr != nil && (e.Tokens.Depth() == 1 || len(e.Buf) > 3*cap(e.Buf)/4)
  127. }
  128. // Flush flushes the buffer to the underlying io.Writer.
  129. // It may append a trailing newline after the top-level value.
  130. func (e *encoderState) Flush() error {
  131. if e.wr == nil || e.avoidFlush() {
  132. return nil
  133. }
  134. // In streaming mode, always emit a newline after the top-level value.
  135. if e.Tokens.Depth() == 1 && !e.Flags.Get(jsonflags.OmitTopLevelNewline) {
  136. e.Buf = append(e.Buf, '\n')
  137. }
  138. // Inform objectNameStack that we are about to flush the buffer content.
  139. e.Names.copyQuotedBuffer(e.Buf)
  140. // Specialize bytes.Buffer for better performance.
  141. if bb, ok := e.wr.(*bytes.Buffer); ok {
  142. // If e.buf already aliases the internal buffer of bb,
  143. // then the Write call simply increments the internal offset,
  144. // otherwise Write operates as expected.
  145. // See https://go.dev/issue/42986.
  146. n, _ := bb.Write(e.Buf) // never fails unless bb is nil
  147. e.baseOffset += int64(n)
  148. // If the internal buffer of bytes.Buffer is too small,
  149. // append operations elsewhere in the Encoder may grow the buffer.
  150. // This would be semantically correct, but hurts performance.
  151. // As such, ensure 25% of the current length is always available
  152. // to reduce the probability that other appends must allocate.
  153. if avail := bb.Available(); avail < bb.Len()/4 {
  154. bb.Grow(avail + 1)
  155. }
  156. e.Buf = bb.AvailableBuffer()
  157. return nil
  158. }
  159. // Flush the internal buffer to the underlying io.Writer.
  160. n, err := e.wr.Write(e.Buf)
  161. e.baseOffset += int64(n)
  162. if err != nil {
  163. // In the event of an error, preserve the unflushed portion.
  164. // Thus, write errors aren't fatal so long as the io.Writer
  165. // maintains consistent state after errors.
  166. if n > 0 {
  167. e.Buf = e.Buf[:copy(e.Buf, e.Buf[n:])]
  168. }
  169. return &ioError{action: "write", err: err}
  170. }
  171. e.Buf = e.Buf[:0]
  172. // Check whether to grow the buffer.
  173. // Note that cap(e.buf) may already exceed maxBufferSize since
  174. // an append elsewhere already grew it to store a large token.
  175. const maxBufferSize = 4 << 10
  176. const growthSizeFactor = 2 // higher value is faster
  177. const growthRateFactor = 2 // higher value is slower
  178. // By default, grow if below the maximum buffer size.
  179. grow := cap(e.Buf) <= maxBufferSize/growthSizeFactor
  180. // Growing can be expensive, so only grow
  181. // if a sufficient number of bytes have been processed.
  182. grow = grow && int64(cap(e.Buf)) < e.previousOffsetEnd()/growthRateFactor
  183. if grow {
  184. e.Buf = make([]byte, 0, cap(e.Buf)*growthSizeFactor)
  185. }
  186. return nil
  187. }
  188. func (d *encodeBuffer) offsetAt(pos int) int64 { return d.baseOffset + int64(pos) }
  189. func (e *encodeBuffer) previousOffsetEnd() int64 { return e.baseOffset + int64(len(e.Buf)) }
  190. func (e *encodeBuffer) unflushedBuffer() []byte { return e.Buf }
  191. // avoidFlush indicates whether to avoid flushing to ensure there is always
  192. // enough in the buffer to unwrite the last object member if it were empty.
  193. func (e *encoderState) avoidFlush() bool {
  194. switch {
  195. case e.Tokens.Last.Length() == 0:
  196. // Never flush after ObjectStart or ArrayStart since we don't know yet
  197. // if the object or array will end up being empty.
  198. return true
  199. case e.Tokens.Last.needObjectValue():
  200. // Never flush before the object value since we don't know yet
  201. // if the object value will end up being empty.
  202. return true
  203. case e.Tokens.Last.NeedObjectName() && len(e.Buf) >= 2:
  204. // Never flush after the object value if it does turn out to be empty.
  205. switch string(e.Buf[len(e.Buf)-2:]) {
  206. case `ll`, `""`, `{}`, `[]`: // last two bytes of every empty value
  207. return true
  208. }
  209. }
  210. return false
  211. }
  212. // UnwriteEmptyObjectMember unwrites the last object member if it is empty
  213. // and reports whether it performed an unwrite operation.
  214. func (e *encoderState) UnwriteEmptyObjectMember(prevName *string) bool {
  215. if last := e.Tokens.Last; !last.isObject() || !last.NeedObjectName() || last.Length() == 0 {
  216. panic("BUG: must be called on an object after writing a value")
  217. }
  218. // The flushing logic is modified to never flush a trailing empty value.
  219. // The encoder never writes trailing whitespace eagerly.
  220. b := e.unflushedBuffer()
  221. // Detect whether the last value was empty.
  222. var n int
  223. if len(b) >= 3 {
  224. switch string(b[len(b)-2:]) {
  225. case "ll": // last two bytes of `null`
  226. n = len(`null`)
  227. case `""`:
  228. // It is possible for a non-empty string to have `""` as a suffix
  229. // if the second to the last quote was escaped.
  230. if b[len(b)-3] == '\\' {
  231. return false // e.g., `"\""` is not empty
  232. }
  233. n = len(`""`)
  234. case `{}`:
  235. n = len(`{}`)
  236. case `[]`:
  237. n = len(`[]`)
  238. }
  239. }
  240. if n == 0 {
  241. return false
  242. }
  243. // Unwrite the value, whitespace, colon, name, whitespace, and comma.
  244. b = b[:len(b)-n]
  245. b = jsonwire.TrimSuffixWhitespace(b)
  246. b = jsonwire.TrimSuffixByte(b, ':')
  247. b = jsonwire.TrimSuffixString(b)
  248. b = jsonwire.TrimSuffixWhitespace(b)
  249. b = jsonwire.TrimSuffixByte(b, ',')
  250. e.Buf = b // store back truncated unflushed buffer
  251. // Undo state changes.
  252. e.Tokens.Last.decrement() // for object member value
  253. e.Tokens.Last.decrement() // for object member name
  254. if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
  255. if e.Tokens.Last.isActiveNamespace() {
  256. e.Namespaces.Last().removeLast()
  257. }
  258. }
  259. e.Names.clearLast()
  260. if prevName != nil {
  261. e.Names.copyQuotedBuffer(e.Buf) // required by objectNameStack.replaceLastUnquotedName
  262. e.Names.replaceLastUnquotedName(*prevName)
  263. }
  264. return true
  265. }
  266. // UnwriteOnlyObjectMemberName unwrites the only object member name
  267. // and returns the unquoted name.
  268. func (e *encoderState) UnwriteOnlyObjectMemberName() string {
  269. if last := e.Tokens.Last; !last.isObject() || last.Length() != 1 {
  270. panic("BUG: must be called on an object after writing first name")
  271. }
  272. // Unwrite the name and whitespace.
  273. b := jsonwire.TrimSuffixString(e.Buf)
  274. isVerbatim := bytes.IndexByte(e.Buf[len(b):], '\\') < 0
  275. name := string(jsonwire.UnquoteMayCopy(e.Buf[len(b):], isVerbatim))
  276. e.Buf = jsonwire.TrimSuffixWhitespace(b)
  277. // Undo state changes.
  278. e.Tokens.Last.decrement()
  279. if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
  280. if e.Tokens.Last.isActiveNamespace() {
  281. e.Namespaces.Last().removeLast()
  282. }
  283. }
  284. e.Names.clearLast()
  285. return name
  286. }
  287. // WriteToken writes the next token and advances the internal write offset.
  288. //
  289. // The provided token kind must be consistent with the JSON grammar.
  290. // For example, it is an error to provide a number when the encoder
  291. // is expecting an object name (which is always a string), or
  292. // to provide an end object delimiter when the encoder is finishing an array.
  293. // If the provided token is invalid, then it reports a [SyntacticError] and
  294. // the internal state remains unchanged. The offset reported
  295. // in [SyntacticError] will be relative to the [Encoder.OutputOffset].
  296. func (e *Encoder) WriteToken(t Token) error {
  297. return e.s.WriteToken(t)
  298. }
  299. func (e *encoderState) WriteToken(t Token) error {
  300. k := t.Kind()
  301. b := e.Buf // use local variable to avoid mutating e in case of error
  302. // Append any delimiters or optional whitespace.
  303. b = e.Tokens.MayAppendDelim(b, k)
  304. if e.Flags.Get(jsonflags.AnyWhitespace) {
  305. b = e.appendWhitespace(b, k)
  306. }
  307. pos := len(b) // offset before the token
  308. // Append the token to the output and to the state machine.
  309. var err error
  310. switch k {
  311. case 'n':
  312. b = append(b, "null"...)
  313. err = e.Tokens.appendLiteral()
  314. case 'f':
  315. b = append(b, "false"...)
  316. err = e.Tokens.appendLiteral()
  317. case 't':
  318. b = append(b, "true"...)
  319. err = e.Tokens.appendLiteral()
  320. case '"':
  321. if b, err = t.appendString(b, &e.Flags); err != nil {
  322. break
  323. }
  324. if e.Tokens.Last.NeedObjectName() {
  325. if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
  326. if !e.Tokens.Last.isValidNamespace() {
  327. err = errInvalidNamespace
  328. break
  329. }
  330. if e.Tokens.Last.isActiveNamespace() && !e.Namespaces.Last().insertQuoted(b[pos:], false) {
  331. err = wrapWithObjectName(ErrDuplicateName, b[pos:])
  332. break
  333. }
  334. }
  335. e.Names.ReplaceLastQuotedOffset(pos) // only replace if insertQuoted succeeds
  336. }
  337. err = e.Tokens.appendString()
  338. case '0':
  339. if b, err = t.appendNumber(b, &e.Flags); err != nil {
  340. break
  341. }
  342. err = e.Tokens.appendNumber()
  343. case '{':
  344. b = append(b, '{')
  345. if err = e.Tokens.pushObject(); err != nil {
  346. break
  347. }
  348. e.Names.push()
  349. if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
  350. e.Namespaces.push()
  351. }
  352. case '}':
  353. b = append(b, '}')
  354. if err = e.Tokens.popObject(); err != nil {
  355. break
  356. }
  357. e.Names.pop()
  358. if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
  359. e.Namespaces.pop()
  360. }
  361. case '[':
  362. b = append(b, '[')
  363. err = e.Tokens.pushArray()
  364. case ']':
  365. b = append(b, ']')
  366. err = e.Tokens.popArray()
  367. default:
  368. err = errInvalidToken
  369. }
  370. if err != nil {
  371. return wrapSyntacticError(e, err, pos, +1)
  372. }
  373. // Finish off the buffer and store it back into e.
  374. e.Buf = b
  375. if e.NeedFlush() {
  376. return e.Flush()
  377. }
  378. return nil
  379. }
  380. // AppendRaw appends either a raw string (without double quotes) or number.
  381. // Specify safeASCII if the string output is guaranteed to be ASCII
  382. // without any characters (including '<', '>', and '&') that need escaping,
  383. // otherwise this will validate whether the string needs escaping.
  384. // The appended bytes for a JSON number must be valid.
  385. //
  386. // This is a specialized implementation of Encoder.WriteValue
  387. // that allows appending directly into the buffer.
  388. // It is only called from marshal logic in the "json" package.
  389. func (e *encoderState) AppendRaw(k Kind, safeASCII bool, appendFn func([]byte) ([]byte, error)) error {
  390. b := e.Buf // use local variable to avoid mutating e in case of error
  391. // Append any delimiters or optional whitespace.
  392. b = e.Tokens.MayAppendDelim(b, k)
  393. if e.Flags.Get(jsonflags.AnyWhitespace) {
  394. b = e.appendWhitespace(b, k)
  395. }
  396. pos := len(b) // offset before the token
  397. var err error
  398. switch k {
  399. case '"':
  400. // Append directly into the encoder buffer by assuming that
  401. // most of the time none of the characters need escaping.
  402. b = append(b, '"')
  403. if b, err = appendFn(b); err != nil {
  404. return err
  405. }
  406. b = append(b, '"')
  407. // Check whether we need to escape the string and if necessary
  408. // copy it to a scratch buffer and then escape it back.
  409. isVerbatim := safeASCII || !jsonwire.NeedEscape(b[pos+len(`"`):len(b)-len(`"`)])
  410. if !isVerbatim {
  411. var err error
  412. b2 := append(e.unusedCache, b[pos+len(`"`):len(b)-len(`"`)]...)
  413. b, err = jsonwire.AppendQuote(b[:pos], string(b2), &e.Flags)
  414. e.unusedCache = b2[:0]
  415. if err != nil {
  416. return wrapSyntacticError(e, err, pos, +1)
  417. }
  418. }
  419. // Update the state machine.
  420. if e.Tokens.Last.NeedObjectName() {
  421. if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
  422. if !e.Tokens.Last.isValidNamespace() {
  423. return wrapSyntacticError(e, err, pos, +1)
  424. }
  425. if e.Tokens.Last.isActiveNamespace() && !e.Namespaces.Last().insertQuoted(b[pos:], isVerbatim) {
  426. err = wrapWithObjectName(ErrDuplicateName, b[pos:])
  427. return wrapSyntacticError(e, err, pos, +1)
  428. }
  429. }
  430. e.Names.ReplaceLastQuotedOffset(pos) // only replace if insertQuoted succeeds
  431. }
  432. if err := e.Tokens.appendString(); err != nil {
  433. return wrapSyntacticError(e, err, pos, +1)
  434. }
  435. case '0':
  436. if b, err = appendFn(b); err != nil {
  437. return err
  438. }
  439. if err := e.Tokens.appendNumber(); err != nil {
  440. return wrapSyntacticError(e, err, pos, +1)
  441. }
  442. default:
  443. panic("BUG: invalid kind")
  444. }
  445. // Finish off the buffer and store it back into e.
  446. e.Buf = b
  447. if e.NeedFlush() {
  448. return e.Flush()
  449. }
  450. return nil
  451. }
  452. // WriteValue writes the next raw value and advances the internal write offset.
  453. // The Encoder does not simply copy the provided value verbatim, but
  454. // parses it to ensure that it is syntactically valid and reformats it
  455. // according to how the Encoder is configured to format whitespace and strings.
  456. // If [AllowInvalidUTF8] is specified, then any invalid UTF-8 is mangled
  457. // as the Unicode replacement character, U+FFFD.
  458. //
  459. // The provided value kind must be consistent with the JSON grammar
  460. // (see examples on [Encoder.WriteToken]). If the provided value is invalid,
  461. // then it reports a [SyntacticError] and the internal state remains unchanged.
  462. // The offset reported in [SyntacticError] will be relative to the
  463. // [Encoder.OutputOffset] plus the offset into v of any encountered syntax error.
  464. func (e *Encoder) WriteValue(v Value) error {
  465. return e.s.WriteValue(v)
  466. }
  467. func (e *encoderState) WriteValue(v Value) error {
  468. e.maxValue |= len(v) // bitwise OR is a fast approximation of max
  469. k := v.Kind()
  470. b := e.Buf // use local variable to avoid mutating e in case of error
  471. // Append any delimiters or optional whitespace.
  472. b = e.Tokens.MayAppendDelim(b, k)
  473. if e.Flags.Get(jsonflags.AnyWhitespace) {
  474. b = e.appendWhitespace(b, k)
  475. }
  476. pos := len(b) // offset before the value
  477. // Append the value the output.
  478. var n int
  479. n += jsonwire.ConsumeWhitespace(v[n:])
  480. b, m, err := e.reformatValue(b, v[n:], e.Tokens.Depth())
  481. if err != nil {
  482. return wrapSyntacticError(e, err, pos+n+m, +1)
  483. }
  484. n += m
  485. n += jsonwire.ConsumeWhitespace(v[n:])
  486. if len(v) > n {
  487. err = jsonwire.NewInvalidCharacterError(v[n:], "after top-level value")
  488. return wrapSyntacticError(e, err, pos+n, 0)
  489. }
  490. // Append the kind to the state machine.
  491. switch k {
  492. case 'n', 'f', 't':
  493. err = e.Tokens.appendLiteral()
  494. case '"':
  495. if e.Tokens.Last.NeedObjectName() {
  496. if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
  497. if !e.Tokens.Last.isValidNamespace() {
  498. err = errInvalidNamespace
  499. break
  500. }
  501. if e.Tokens.Last.isActiveNamespace() && !e.Namespaces.Last().insertQuoted(b[pos:], false) {
  502. err = wrapWithObjectName(ErrDuplicateName, b[pos:])
  503. break
  504. }
  505. }
  506. e.Names.ReplaceLastQuotedOffset(pos) // only replace if insertQuoted succeeds
  507. }
  508. err = e.Tokens.appendString()
  509. case '0':
  510. err = e.Tokens.appendNumber()
  511. case '{':
  512. if err = e.Tokens.pushObject(); err != nil {
  513. break
  514. }
  515. if err = e.Tokens.popObject(); err != nil {
  516. panic("BUG: popObject should never fail immediately after pushObject: " + err.Error())
  517. }
  518. if e.Flags.Get(jsonflags.ReorderRawObjects) {
  519. mustReorderObjects(b[pos:])
  520. }
  521. case '[':
  522. if err = e.Tokens.pushArray(); err != nil {
  523. break
  524. }
  525. if err = e.Tokens.popArray(); err != nil {
  526. panic("BUG: popArray should never fail immediately after pushArray: " + err.Error())
  527. }
  528. if e.Flags.Get(jsonflags.ReorderRawObjects) {
  529. mustReorderObjects(b[pos:])
  530. }
  531. }
  532. if err != nil {
  533. return wrapSyntacticError(e, err, pos, +1)
  534. }
  535. // Finish off the buffer and store it back into e.
  536. e.Buf = b
  537. if e.NeedFlush() {
  538. return e.Flush()
  539. }
  540. return nil
  541. }
  542. // CountNextDelimWhitespace counts the number of bytes of delimiter and
  543. // whitespace bytes assuming the upcoming token is a JSON value.
  544. // This method is used for error reporting at the semantic layer.
  545. func (e *encoderState) CountNextDelimWhitespace() (n int) {
  546. const next = Kind('"') // arbitrary kind as next JSON value
  547. delim := e.Tokens.needDelim(next)
  548. if delim > 0 {
  549. n += len(",") | len(":")
  550. }
  551. if delim == ':' {
  552. if e.Flags.Get(jsonflags.SpaceAfterColon) {
  553. n += len(" ")
  554. }
  555. } else {
  556. if delim == ',' && e.Flags.Get(jsonflags.SpaceAfterComma) {
  557. n += len(" ")
  558. }
  559. if e.Flags.Get(jsonflags.Multiline) {
  560. if m := e.Tokens.NeedIndent(next); m > 0 {
  561. n += len("\n") + len(e.IndentPrefix) + (m-1)*len(e.Indent)
  562. }
  563. }
  564. }
  565. return n
  566. }
  567. // appendWhitespace appends whitespace that immediately precedes the next token.
  568. func (e *encoderState) appendWhitespace(b []byte, next Kind) []byte {
  569. if delim := e.Tokens.needDelim(next); delim == ':' {
  570. if e.Flags.Get(jsonflags.SpaceAfterColon) {
  571. b = append(b, ' ')
  572. }
  573. } else {
  574. if delim == ',' && e.Flags.Get(jsonflags.SpaceAfterComma) {
  575. b = append(b, ' ')
  576. }
  577. if e.Flags.Get(jsonflags.Multiline) {
  578. b = e.AppendIndent(b, e.Tokens.NeedIndent(next))
  579. }
  580. }
  581. return b
  582. }
  583. // AppendIndent appends the appropriate number of indentation characters
  584. // for the current nested level, n.
  585. func (e *encoderState) AppendIndent(b []byte, n int) []byte {
  586. if n == 0 {
  587. return b
  588. }
  589. b = append(b, '\n')
  590. b = append(b, e.IndentPrefix...)
  591. for ; n > 1; n-- {
  592. b = append(b, e.Indent...)
  593. }
  594. return b
  595. }
  596. // reformatValue parses a JSON value from the start of src and
  597. // appends it to the end of dst, reformatting whitespace and strings as needed.
  598. // It returns the extended dst buffer and the number of consumed input bytes.
  599. func (e *encoderState) reformatValue(dst []byte, src Value, depth int) ([]byte, int, error) {
  600. // TODO: Should this update ValueFlags as input?
  601. if len(src) == 0 {
  602. return dst, 0, io.ErrUnexpectedEOF
  603. }
  604. switch k := Kind(src[0]).normalize(); k {
  605. case 'n':
  606. if jsonwire.ConsumeNull(src) == 0 {
  607. n, err := jsonwire.ConsumeLiteral(src, "null")
  608. return dst, n, err
  609. }
  610. return append(dst, "null"...), len("null"), nil
  611. case 'f':
  612. if jsonwire.ConsumeFalse(src) == 0 {
  613. n, err := jsonwire.ConsumeLiteral(src, "false")
  614. return dst, n, err
  615. }
  616. return append(dst, "false"...), len("false"), nil
  617. case 't':
  618. if jsonwire.ConsumeTrue(src) == 0 {
  619. n, err := jsonwire.ConsumeLiteral(src, "true")
  620. return dst, n, err
  621. }
  622. return append(dst, "true"...), len("true"), nil
  623. case '"':
  624. if n := jsonwire.ConsumeSimpleString(src); n > 0 {
  625. dst = append(dst, src[:n]...) // copy simple strings verbatim
  626. return dst, n, nil
  627. }
  628. return jsonwire.ReformatString(dst, src, &e.Flags)
  629. case '0':
  630. if n := jsonwire.ConsumeSimpleNumber(src); n > 0 && !e.Flags.Get(jsonflags.CanonicalizeNumbers) {
  631. dst = append(dst, src[:n]...) // copy simple numbers verbatim
  632. return dst, n, nil
  633. }
  634. return jsonwire.ReformatNumber(dst, src, &e.Flags)
  635. case '{':
  636. return e.reformatObject(dst, src, depth)
  637. case '[':
  638. return e.reformatArray(dst, src, depth)
  639. default:
  640. return dst, 0, jsonwire.NewInvalidCharacterError(src, "at start of value")
  641. }
  642. }
  643. // reformatObject parses a JSON object from the start of src and
  644. // appends it to the end of src, reformatting whitespace and strings as needed.
  645. // It returns the extended dst buffer and the number of consumed input bytes.
  646. func (e *encoderState) reformatObject(dst []byte, src Value, depth int) ([]byte, int, error) {
  647. // Append object start.
  648. if len(src) == 0 || src[0] != '{' {
  649. panic("BUG: reformatObject must be called with a buffer that starts with '{'")
  650. } else if depth == maxNestingDepth+1 {
  651. return dst, 0, errMaxDepth
  652. }
  653. dst = append(dst, '{')
  654. n := len("{")
  655. // Append (possible) object end.
  656. n += jsonwire.ConsumeWhitespace(src[n:])
  657. if uint(len(src)) <= uint(n) {
  658. return dst, n, io.ErrUnexpectedEOF
  659. }
  660. if src[n] == '}' {
  661. dst = append(dst, '}')
  662. n += len("}")
  663. return dst, n, nil
  664. }
  665. var err error
  666. var names *objectNamespace
  667. if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
  668. e.Namespaces.push()
  669. defer e.Namespaces.pop()
  670. names = e.Namespaces.Last()
  671. }
  672. depth++
  673. for {
  674. // Append optional newline and indentation.
  675. if e.Flags.Get(jsonflags.Multiline) {
  676. dst = e.AppendIndent(dst, depth)
  677. }
  678. // Append object name.
  679. n += jsonwire.ConsumeWhitespace(src[n:])
  680. if uint(len(src)) <= uint(n) {
  681. return dst, n, io.ErrUnexpectedEOF
  682. }
  683. m := jsonwire.ConsumeSimpleString(src[n:])
  684. isVerbatim := m > 0
  685. if isVerbatim {
  686. dst = append(dst, src[n:n+m]...)
  687. } else {
  688. dst, m, err = jsonwire.ReformatString(dst, src[n:], &e.Flags)
  689. if err != nil {
  690. return dst, n + m, err
  691. }
  692. }
  693. quotedName := src[n : n+m]
  694. if !e.Flags.Get(jsonflags.AllowDuplicateNames) && !names.insertQuoted(quotedName, isVerbatim) {
  695. return dst, n, wrapWithObjectName(ErrDuplicateName, quotedName)
  696. }
  697. n += m
  698. // Append colon.
  699. n += jsonwire.ConsumeWhitespace(src[n:])
  700. if uint(len(src)) <= uint(n) {
  701. return dst, n, wrapWithObjectName(io.ErrUnexpectedEOF, quotedName)
  702. }
  703. if src[n] != ':' {
  704. err = jsonwire.NewInvalidCharacterError(src[n:], "after object name (expecting ':')")
  705. return dst, n, wrapWithObjectName(err, quotedName)
  706. }
  707. dst = append(dst, ':')
  708. n += len(":")
  709. if e.Flags.Get(jsonflags.SpaceAfterColon) {
  710. dst = append(dst, ' ')
  711. }
  712. // Append object value.
  713. n += jsonwire.ConsumeWhitespace(src[n:])
  714. if uint(len(src)) <= uint(n) {
  715. return dst, n, wrapWithObjectName(io.ErrUnexpectedEOF, quotedName)
  716. }
  717. dst, m, err = e.reformatValue(dst, src[n:], depth)
  718. if err != nil {
  719. return dst, n + m, wrapWithObjectName(err, quotedName)
  720. }
  721. n += m
  722. // Append comma or object end.
  723. n += jsonwire.ConsumeWhitespace(src[n:])
  724. if uint(len(src)) <= uint(n) {
  725. return dst, n, io.ErrUnexpectedEOF
  726. }
  727. switch src[n] {
  728. case ',':
  729. dst = append(dst, ',')
  730. if e.Flags.Get(jsonflags.SpaceAfterComma) {
  731. dst = append(dst, ' ')
  732. }
  733. n += len(",")
  734. continue
  735. case '}':
  736. if e.Flags.Get(jsonflags.Multiline) {
  737. dst = e.AppendIndent(dst, depth-1)
  738. }
  739. dst = append(dst, '}')
  740. n += len("}")
  741. return dst, n, nil
  742. default:
  743. return dst, n, jsonwire.NewInvalidCharacterError(src[n:], "after object value (expecting ',' or '}')")
  744. }
  745. }
  746. }
  747. // reformatArray parses a JSON array from the start of src and
  748. // appends it to the end of dst, reformatting whitespace and strings as needed.
  749. // It returns the extended dst buffer and the number of consumed input bytes.
  750. func (e *encoderState) reformatArray(dst []byte, src Value, depth int) ([]byte, int, error) {
  751. // Append array start.
  752. if len(src) == 0 || src[0] != '[' {
  753. panic("BUG: reformatArray must be called with a buffer that starts with '['")
  754. } else if depth == maxNestingDepth+1 {
  755. return dst, 0, errMaxDepth
  756. }
  757. dst = append(dst, '[')
  758. n := len("[")
  759. // Append (possible) array end.
  760. n += jsonwire.ConsumeWhitespace(src[n:])
  761. if uint(len(src)) <= uint(n) {
  762. return dst, n, io.ErrUnexpectedEOF
  763. }
  764. if src[n] == ']' {
  765. dst = append(dst, ']')
  766. n += len("]")
  767. return dst, n, nil
  768. }
  769. var idx int64
  770. var err error
  771. depth++
  772. for {
  773. // Append optional newline and indentation.
  774. if e.Flags.Get(jsonflags.Multiline) {
  775. dst = e.AppendIndent(dst, depth)
  776. }
  777. // Append array value.
  778. n += jsonwire.ConsumeWhitespace(src[n:])
  779. if uint(len(src)) <= uint(n) {
  780. return dst, n, io.ErrUnexpectedEOF
  781. }
  782. var m int
  783. dst, m, err = e.reformatValue(dst, src[n:], depth)
  784. if err != nil {
  785. return dst, n + m, wrapWithArrayIndex(err, idx)
  786. }
  787. n += m
  788. // Append comma or array end.
  789. n += jsonwire.ConsumeWhitespace(src[n:])
  790. if uint(len(src)) <= uint(n) {
  791. return dst, n, io.ErrUnexpectedEOF
  792. }
  793. switch src[n] {
  794. case ',':
  795. dst = append(dst, ',')
  796. if e.Flags.Get(jsonflags.SpaceAfterComma) {
  797. dst = append(dst, ' ')
  798. }
  799. n += len(",")
  800. idx++
  801. continue
  802. case ']':
  803. if e.Flags.Get(jsonflags.Multiline) {
  804. dst = e.AppendIndent(dst, depth-1)
  805. }
  806. dst = append(dst, ']')
  807. n += len("]")
  808. return dst, n, nil
  809. default:
  810. return dst, n, jsonwire.NewInvalidCharacterError(src[n:], "after array value (expecting ',' or ']')")
  811. }
  812. }
  813. }
  814. // OutputOffset returns the current output byte offset. It gives the location
  815. // of the next byte immediately after the most recently written token or value.
  816. // The number of bytes actually written to the underlying [io.Writer] may be less
  817. // than this offset due to internal buffering effects.
  818. func (e *Encoder) OutputOffset() int64 {
  819. return e.s.previousOffsetEnd()
  820. }
  821. // UnusedBuffer returns a zero-length buffer with a possible non-zero capacity.
  822. // This buffer is intended to be used to populate a [Value]
  823. // being passed to an immediately succeeding [Encoder.WriteValue] call.
  824. //
  825. // Example usage:
  826. //
  827. // b := d.UnusedBuffer()
  828. // b = append(b, '"')
  829. // b = appendString(b, v) // append the string formatting of v
  830. // b = append(b, '"')
  831. // ... := d.WriteValue(b)
  832. //
  833. // It is the user's responsibility to ensure that the value is valid JSON.
  834. func (e *Encoder) UnusedBuffer() []byte {
  835. // NOTE: We don't return e.buf[len(e.buf):cap(e.buf)] since WriteValue would
  836. // need to take special care to avoid mangling the data while reformatting.
  837. // WriteValue can't easily identify whether the input Value aliases e.buf
  838. // without using unsafe.Pointer. Thus, we just return a different buffer.
  839. // Should this ever alias e.buf, we need to consider how it operates with
  840. // the specialized performance optimization for bytes.Buffer.
  841. n := 1 << bits.Len(uint(e.s.maxValue|63)) // fast approximation for max length
  842. if cap(e.s.unusedCache) < n {
  843. e.s.unusedCache = make([]byte, 0, n)
  844. }
  845. return e.s.unusedCache
  846. }
  847. // StackDepth returns the depth of the state machine for written JSON data.
  848. // Each level on the stack represents a nested JSON object or array.
  849. // It is incremented whenever an [ObjectStart] or [ArrayStart] token is encountered
  850. // and decremented whenever an [ObjectEnd] or [ArrayEnd] token is encountered.
  851. // The depth is zero-indexed, where zero represents the top-level JSON value.
  852. func (e *Encoder) StackDepth() int {
  853. // NOTE: Keep in sync with Decoder.StackDepth.
  854. return e.s.Tokens.Depth() - 1
  855. }
  856. // StackIndex returns information about the specified stack level.
  857. // It must be a number between 0 and [Encoder.StackDepth], inclusive.
  858. // For each level, it reports the kind:
  859. //
  860. // - 0 for a level of zero,
  861. // - '{' for a level representing a JSON object, and
  862. // - '[' for a level representing a JSON array.
  863. //
  864. // It also reports the length of that JSON object or array.
  865. // Each name and value in a JSON object is counted separately,
  866. // so the effective number of members would be half the length.
  867. // A complete JSON object must have an even length.
  868. func (e *Encoder) StackIndex(i int) (Kind, int64) {
  869. // NOTE: Keep in sync with Decoder.StackIndex.
  870. switch s := e.s.Tokens.index(i); {
  871. case i > 0 && s.isObject():
  872. return '{', s.Length()
  873. case i > 0 && s.isArray():
  874. return '[', s.Length()
  875. default:
  876. return 0, s.Length()
  877. }
  878. }
  879. // StackPointer returns a JSON Pointer (RFC 6901) to the most recently written value.
  880. func (e *Encoder) StackPointer() Pointer {
  881. return Pointer(e.s.AppendStackPointer(nil, -1))
  882. }
  883. func (e *encoderState) AppendStackPointer(b []byte, where int) []byte {
  884. e.Names.copyQuotedBuffer(e.Buf)
  885. return e.state.appendStackPointer(b, where)
  886. }