token.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525
  1. // Copyright 2020 The Go Authors. All rights reserved.
  2. // Use of this source code is governed by a BSD-style
  3. // license that can be found in the LICENSE file.
  4. package jsontext
  5. import (
  6. "bytes"
  7. "errors"
  8. "math"
  9. "strconv"
  10. "github.com/go-json-experiment/json/internal/jsonflags"
  11. "github.com/go-json-experiment/json/internal/jsonwire"
  12. )
  13. // NOTE: Token is analogous to v1 json.Token.
  14. const (
  15. maxInt64 = math.MaxInt64
  16. minInt64 = math.MinInt64
  17. maxUint64 = math.MaxUint64
  18. minUint64 = 0 // for consistency and readability purposes
  19. invalidTokenPanic = "invalid jsontext.Token; it has been voided by a subsequent json.Decoder call"
  20. )
  21. var errInvalidToken = errors.New("invalid jsontext.Token")
  22. // Token represents a lexical JSON token, which may be one of the following:
  23. // - a JSON literal (i.e., null, true, or false)
  24. // - a JSON string (e.g., "hello, world!")
  25. // - a JSON number (e.g., 123.456)
  26. // - a start or end delimiter for a JSON object (i.e., { or } )
  27. // - a start or end delimiter for a JSON array (i.e., [ or ] )
  28. //
  29. // A Token cannot represent entire array or object values, while a [Value] can.
  30. // There is no Token to represent commas and colons since
  31. // these structural tokens can be inferred from the surrounding context.
  32. type Token struct {
  33. nonComparable
  34. // Tokens can exist in either a "raw" or an "exact" form.
  35. // Tokens produced by the Decoder are in the "raw" form.
  36. // Tokens returned by constructors are usually in the "exact" form.
  37. // The Encoder accepts Tokens in either the "raw" or "exact" form.
  38. //
  39. // The following chart shows the possible values for each Token type:
  40. // ╔═════════════════╦════════════╤════════════╤════════════╗
  41. // ║ Token type ║ raw field │ str field │ num field ║
  42. // ╠═════════════════╬════════════╪════════════╪════════════╣
  43. // ║ null (raw) ║ "null" │ "" │ 0 ║
  44. // ║ false (raw) ║ "false" │ "" │ 0 ║
  45. // ║ true (raw) ║ "true" │ "" │ 0 ║
  46. // ║ string (raw) ║ non-empty │ "" │ offset ║
  47. // ║ string (string) ║ nil │ non-empty │ 0 ║
  48. // ║ number (raw) ║ non-empty │ "" │ offset ║
  49. // ║ number (float) ║ nil │ "f" │ non-zero ║
  50. // ║ number (int64) ║ nil │ "i" │ non-zero ║
  51. // ║ number (uint64) ║ nil │ "u" │ non-zero ║
  52. // ║ object (delim) ║ "{" or "}" │ "" │ 0 ║
  53. // ║ array (delim) ║ "[" or "]" │ "" │ 0 ║
  54. // ╚═════════════════╩════════════╧════════════╧════════════╝
  55. //
  56. // Notes:
  57. // - For tokens stored in "raw" form, the num field contains the
  58. // absolute offset determined by raw.previousOffsetStart().
  59. // The buffer itself is stored in raw.previousBuffer().
  60. // - JSON literals and structural characters are always in the "raw" form.
  61. // - JSON strings and numbers can be in either "raw" or "exact" forms.
  62. // - The exact zero value of JSON strings and numbers in the "exact" forms
  63. // have ambiguous representation. Thus, they are always represented
  64. // in the "raw" form.
  65. // raw contains a reference to the raw decode buffer.
  66. // If non-nil, then its value takes precedence over str and num.
  67. // It is only valid if num == raw.previousOffsetStart().
  68. raw *decodeBuffer
  69. // str is the unescaped JSON string if num is zero.
  70. // Otherwise, it is "f", "i", or "u" if num should be interpreted
  71. // as a float64, int64, or uint64, respectively.
  72. str string
  73. // num is a float64, int64, or uint64 stored as a uint64 value.
  74. // It is non-zero for any JSON number in the "exact" form.
  75. num uint64
  76. }
  77. // TODO: Does representing 1-byte delimiters as *decodeBuffer cause performance issues?
  78. var (
  79. Null Token = rawToken("null")
  80. False Token = rawToken("false")
  81. True Token = rawToken("true")
  82. ObjectStart Token = rawToken("{")
  83. ObjectEnd Token = rawToken("}")
  84. ArrayStart Token = rawToken("[")
  85. ArrayEnd Token = rawToken("]")
  86. zeroString Token = rawToken(`""`)
  87. zeroNumber Token = rawToken(`0`)
  88. nanString Token = String("NaN")
  89. pinfString Token = String("Infinity")
  90. ninfString Token = String("-Infinity")
  91. )
  92. func rawToken(s string) Token {
  93. return Token{raw: &decodeBuffer{buf: []byte(s), prevStart: 0, prevEnd: len(s)}}
  94. }
  95. // Bool constructs a Token representing a JSON boolean.
  96. func Bool(b bool) Token {
  97. if b {
  98. return True
  99. }
  100. return False
  101. }
  102. // String constructs a Token representing a JSON string.
  103. // The provided string should contain valid UTF-8, otherwise invalid characters
  104. // may be mangled as the Unicode replacement character.
  105. func String(s string) Token {
  106. if len(s) == 0 {
  107. return zeroString
  108. }
  109. return Token{str: s}
  110. }
  111. // Float constructs a Token representing a JSON number.
  112. // The values NaN, +Inf, and -Inf will be represented
  113. // as a JSON string with the values "NaN", "Infinity", and "-Infinity".
  114. func Float(n float64) Token {
  115. switch {
  116. case math.Float64bits(n) == 0:
  117. return zeroNumber
  118. case math.IsNaN(n):
  119. return nanString
  120. case math.IsInf(n, +1):
  121. return pinfString
  122. case math.IsInf(n, -1):
  123. return ninfString
  124. }
  125. return Token{str: "f", num: math.Float64bits(n)}
  126. }
  127. // Int constructs a Token representing a JSON number from an int64.
  128. func Int(n int64) Token {
  129. if n == 0 {
  130. return zeroNumber
  131. }
  132. return Token{str: "i", num: uint64(n)}
  133. }
  134. // Uint constructs a Token representing a JSON number from a uint64.
  135. func Uint(n uint64) Token {
  136. if n == 0 {
  137. return zeroNumber
  138. }
  139. return Token{str: "u", num: uint64(n)}
  140. }
  141. // Clone makes a copy of the Token such that its value remains valid
  142. // even after a subsequent [Decoder.Read] call.
  143. func (t Token) Clone() Token {
  144. // TODO: Allow caller to avoid any allocations?
  145. if raw := t.raw; raw != nil {
  146. // Avoid copying globals.
  147. if t.raw.prevStart == 0 {
  148. switch t.raw {
  149. case Null.raw:
  150. return Null
  151. case False.raw:
  152. return False
  153. case True.raw:
  154. return True
  155. case ObjectStart.raw:
  156. return ObjectStart
  157. case ObjectEnd.raw:
  158. return ObjectEnd
  159. case ArrayStart.raw:
  160. return ArrayStart
  161. case ArrayEnd.raw:
  162. return ArrayEnd
  163. }
  164. }
  165. if uint64(raw.previousOffsetStart()) != t.num {
  166. panic(invalidTokenPanic)
  167. }
  168. buf := bytes.Clone(raw.previousBuffer())
  169. return Token{raw: &decodeBuffer{buf: buf, prevStart: 0, prevEnd: len(buf)}}
  170. }
  171. return t
  172. }
  173. // Bool returns the value for a JSON boolean.
  174. // It panics if the token kind is not a JSON boolean.
  175. func (t Token) Bool() bool {
  176. switch t.raw {
  177. case True.raw:
  178. return true
  179. case False.raw:
  180. return false
  181. default:
  182. panic("invalid JSON token kind: " + t.Kind().String())
  183. }
  184. }
  185. // appendString appends a JSON string to dst and returns it.
  186. // It panics if t is not a JSON string.
  187. func (t Token) appendString(dst []byte, flags *jsonflags.Flags) ([]byte, error) {
  188. if raw := t.raw; raw != nil {
  189. // Handle raw string value.
  190. buf := raw.previousBuffer()
  191. if Kind(buf[0]) == '"' {
  192. if jsonwire.ConsumeSimpleString(buf) == len(buf) {
  193. return append(dst, buf...), nil
  194. }
  195. dst, _, err := jsonwire.ReformatString(dst, buf, flags)
  196. return dst, err
  197. }
  198. } else if len(t.str) != 0 && t.num == 0 {
  199. // Handle exact string value.
  200. return jsonwire.AppendQuote(dst, t.str, flags)
  201. }
  202. panic("invalid JSON token kind: " + t.Kind().String())
  203. }
  204. // String returns the unescaped string value for a JSON string.
  205. // For other JSON kinds, this returns the raw JSON representation.
  206. func (t Token) String() string {
  207. // This is inlinable to take advantage of "function outlining".
  208. // This avoids an allocation for the string(b) conversion
  209. // if the caller does not use the string in an escaping manner.
  210. // See https://blog.filippo.io/efficient-go-apis-with-the-inliner/
  211. s, b := t.string()
  212. if len(b) > 0 {
  213. return string(b)
  214. }
  215. return s
  216. }
  217. func (t Token) string() (string, []byte) {
  218. if raw := t.raw; raw != nil {
  219. if uint64(raw.previousOffsetStart()) != t.num {
  220. panic(invalidTokenPanic)
  221. }
  222. buf := raw.previousBuffer()
  223. if buf[0] == '"' {
  224. // TODO: Preserve ValueFlags in Token?
  225. isVerbatim := jsonwire.ConsumeSimpleString(buf) == len(buf)
  226. return "", jsonwire.UnquoteMayCopy(buf, isVerbatim)
  227. }
  228. // Handle tokens that are not JSON strings for fmt.Stringer.
  229. return "", buf
  230. }
  231. if len(t.str) != 0 && t.num == 0 {
  232. return t.str, nil
  233. }
  234. // Handle tokens that are not JSON strings for fmt.Stringer.
  235. if t.num > 0 {
  236. switch t.str[0] {
  237. case 'f':
  238. return string(jsonwire.AppendFloat(nil, math.Float64frombits(t.num), 64)), nil
  239. case 'i':
  240. return strconv.FormatInt(int64(t.num), 10), nil
  241. case 'u':
  242. return strconv.FormatUint(uint64(t.num), 10), nil
  243. }
  244. }
  245. return "<invalid jsontext.Token>", nil
  246. }
  247. // appendNumber appends a JSON number to dst and returns it.
  248. // It panics if t is not a JSON number.
  249. func (t Token) appendNumber(dst []byte, flags *jsonflags.Flags) ([]byte, error) {
  250. if raw := t.raw; raw != nil {
  251. // Handle raw number value.
  252. buf := raw.previousBuffer()
  253. if Kind(buf[0]).normalize() == '0' {
  254. dst, _, err := jsonwire.ReformatNumber(dst, buf, flags)
  255. return dst, err
  256. }
  257. } else if t.num != 0 {
  258. // Handle exact number value.
  259. switch t.str[0] {
  260. case 'f':
  261. return jsonwire.AppendFloat(dst, math.Float64frombits(t.num), 64), nil
  262. case 'i':
  263. return strconv.AppendInt(dst, int64(t.num), 10), nil
  264. case 'u':
  265. return strconv.AppendUint(dst, uint64(t.num), 10), nil
  266. }
  267. }
  268. panic("invalid JSON token kind: " + t.Kind().String())
  269. }
  270. // Float returns the floating-point value for a JSON number.
  271. // It returns a NaN, +Inf, or -Inf value for any JSON string
  272. // with the values "NaN", "Infinity", or "-Infinity".
  273. // It panics for all other cases.
  274. func (t Token) Float() float64 {
  275. if raw := t.raw; raw != nil {
  276. // Handle raw number value.
  277. if uint64(raw.previousOffsetStart()) != t.num {
  278. panic(invalidTokenPanic)
  279. }
  280. buf := raw.previousBuffer()
  281. if Kind(buf[0]).normalize() == '0' {
  282. fv, _ := jsonwire.ParseFloat(buf, 64)
  283. return fv
  284. }
  285. } else if t.num != 0 {
  286. // Handle exact number value.
  287. switch t.str[0] {
  288. case 'f':
  289. return math.Float64frombits(t.num)
  290. case 'i':
  291. return float64(int64(t.num))
  292. case 'u':
  293. return float64(uint64(t.num))
  294. }
  295. }
  296. // Handle string values with "NaN", "Infinity", or "-Infinity".
  297. if t.Kind() == '"' {
  298. switch t.String() {
  299. case "NaN":
  300. return math.NaN()
  301. case "Infinity":
  302. return math.Inf(+1)
  303. case "-Infinity":
  304. return math.Inf(-1)
  305. }
  306. }
  307. panic("invalid JSON token kind: " + t.Kind().String())
  308. }
  309. // Int returns the signed integer value for a JSON number.
  310. // The fractional component of any number is ignored (truncation toward zero).
  311. // Any number beyond the representation of an int64 will be saturated
  312. // to the closest representable value.
  313. // It panics if the token kind is not a JSON number.
  314. func (t Token) Int() int64 {
  315. if raw := t.raw; raw != nil {
  316. // Handle raw integer value.
  317. if uint64(raw.previousOffsetStart()) != t.num {
  318. panic(invalidTokenPanic)
  319. }
  320. neg := false
  321. buf := raw.previousBuffer()
  322. if len(buf) > 0 && buf[0] == '-' {
  323. neg, buf = true, buf[1:]
  324. }
  325. if numAbs, ok := jsonwire.ParseUint(buf); ok {
  326. if neg {
  327. if numAbs > -minInt64 {
  328. return minInt64
  329. }
  330. return -1 * int64(numAbs)
  331. } else {
  332. if numAbs > +maxInt64 {
  333. return maxInt64
  334. }
  335. return +1 * int64(numAbs)
  336. }
  337. }
  338. } else if t.num != 0 {
  339. // Handle exact integer value.
  340. switch t.str[0] {
  341. case 'i':
  342. return int64(t.num)
  343. case 'u':
  344. if t.num > maxInt64 {
  345. return maxInt64
  346. }
  347. return int64(t.num)
  348. }
  349. }
  350. // Handle JSON number that is a floating-point value.
  351. if t.Kind() == '0' {
  352. switch fv := t.Float(); {
  353. case fv >= maxInt64:
  354. return maxInt64
  355. case fv <= minInt64:
  356. return minInt64
  357. default:
  358. return int64(fv) // truncation toward zero
  359. }
  360. }
  361. panic("invalid JSON token kind: " + t.Kind().String())
  362. }
  363. // Uint returns the unsigned integer value for a JSON number.
  364. // The fractional component of any number is ignored (truncation toward zero).
  365. // Any number beyond the representation of an uint64 will be saturated
  366. // to the closest representable value.
  367. // It panics if the token kind is not a JSON number.
  368. func (t Token) Uint() uint64 {
  369. // NOTE: This accessor returns 0 for any negative JSON number,
  370. // which might be surprising, but is at least consistent with the behavior
  371. // of saturating out-of-bounds numbers to the closest representable number.
  372. if raw := t.raw; raw != nil {
  373. // Handle raw integer value.
  374. if uint64(raw.previousOffsetStart()) != t.num {
  375. panic(invalidTokenPanic)
  376. }
  377. neg := false
  378. buf := raw.previousBuffer()
  379. if len(buf) > 0 && buf[0] == '-' {
  380. neg, buf = true, buf[1:]
  381. }
  382. if num, ok := jsonwire.ParseUint(buf); ok {
  383. if neg {
  384. return minUint64
  385. }
  386. return num
  387. }
  388. } else if t.num != 0 {
  389. // Handle exact integer value.
  390. switch t.str[0] {
  391. case 'u':
  392. return t.num
  393. case 'i':
  394. if int64(t.num) < minUint64 {
  395. return minUint64
  396. }
  397. return uint64(int64(t.num))
  398. }
  399. }
  400. // Handle JSON number that is a floating-point value.
  401. if t.Kind() == '0' {
  402. switch fv := t.Float(); {
  403. case fv >= maxUint64:
  404. return maxUint64
  405. case fv <= minUint64:
  406. return minUint64
  407. default:
  408. return uint64(fv) // truncation toward zero
  409. }
  410. }
  411. panic("invalid JSON token kind: " + t.Kind().String())
  412. }
  413. // Kind returns the token kind.
  414. func (t Token) Kind() Kind {
  415. switch {
  416. case t.raw != nil:
  417. raw := t.raw
  418. if uint64(raw.previousOffsetStart()) != t.num {
  419. panic(invalidTokenPanic)
  420. }
  421. return Kind(t.raw.buf[raw.prevStart]).normalize()
  422. case t.num != 0:
  423. return '0'
  424. case len(t.str) != 0:
  425. return '"'
  426. default:
  427. return invalidKind
  428. }
  429. }
  430. // Kind represents each possible JSON token kind with a single byte,
  431. // which is conveniently the first byte of that kind's grammar
  432. // with the restriction that numbers always be represented with '0':
  433. //
  434. // - 'n': null
  435. // - 'f': false
  436. // - 't': true
  437. // - '"': string
  438. // - '0': number
  439. // - '{': object start
  440. // - '}': object end
  441. // - '[': array start
  442. // - ']': array end
  443. //
  444. // An invalid kind is usually represented using 0,
  445. // but may be non-zero due to invalid JSON data.
  446. type Kind byte
  447. const invalidKind Kind = 0
  448. // String prints the kind in a humanly readable fashion.
  449. func (k Kind) String() string {
  450. switch k {
  451. case 'n':
  452. return "null"
  453. case 'f':
  454. return "false"
  455. case 't':
  456. return "true"
  457. case '"':
  458. return "string"
  459. case '0':
  460. return "number"
  461. case '{':
  462. return "{"
  463. case '}':
  464. return "}"
  465. case '[':
  466. return "["
  467. case ']':
  468. return "]"
  469. default:
  470. return "<invalid jsontext.Kind: " + jsonwire.QuoteRune(string(k)) + ">"
  471. }
  472. }
  473. // normalize coalesces all possible starting characters of a number as just '0'.
  474. func (k Kind) normalize() Kind {
  475. if k == '-' || ('0' <= k && k <= '9') {
  476. return '0'
  477. }
  478. return k
  479. }