parser.go (14420B)
1 // Copyright 2015 Unknwon 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"): you may 4 // not use this file except in compliance with the License. You may obtain 5 // a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11 // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12 // License for the specific language governing permissions and limitations 13 // under the License. 14 15 package ini 16 17 import ( 18 "bufio" 19 "bytes" 20 "fmt" 21 "io" 22 "regexp" 23 "strconv" 24 "strings" 25 "unicode" 26 ) 27 28 const minReaderBufferSize = 4096 29 30 var pythonMultiline = regexp.MustCompile(`^([\t\f ]+)(.*)`) 31 32 type parserOptions struct { 33 IgnoreContinuation bool 34 IgnoreInlineComment bool 35 AllowPythonMultilineValues bool 36 SpaceBeforeInlineComment bool 37 UnescapeValueDoubleQuotes bool 38 UnescapeValueCommentSymbols bool 39 PreserveSurroundedQuote bool 40 DebugFunc DebugFunc 41 ReaderBufferSize int 42 } 43 44 type parser struct { 45 buf *bufio.Reader 46 options parserOptions 47 48 isEOF bool 49 count int 50 comment *bytes.Buffer 51 } 52 53 func (p *parser) debug(format string, args ...interface{}) { 54 if p.options.DebugFunc != nil { 55 p.options.DebugFunc(fmt.Sprintf(format, args...)) 56 } 57 } 58 59 func newParser(r io.Reader, opts parserOptions) *parser { 60 size := opts.ReaderBufferSize 61 if size < minReaderBufferSize { 62 size = minReaderBufferSize 63 } 64 65 return &parser{ 66 buf: bufio.NewReaderSize(r, size), 67 options: opts, 68 count: 1, 69 comment: &bytes.Buffer{}, 70 } 71 } 72 73 // BOM handles header of UTF-8, UTF-16 LE and UTF-16 BE's BOM format. 74 // http://en.wikipedia.org/wiki/Byte_order_mark#Representations_of_byte_order_marks_by_encoding 75 func (p *parser) BOM() error { 76 // Strip every leading BOM: a file may accidentally carry more than one 77 // (e.g. a tool prepending a BOM to already-BOM'd content). Stopping after 78 // the first one leaves the remaining BOM on the next token and makes the 79 // parser fail with a misleading "key-value delimiter not found" (#390). 80 for { 81 mask, err := p.buf.Peek(2) 82 if err != nil && err != io.EOF { 83 return err 84 } else if len(mask) < 2 { 85 return nil 86 } 87 88 switch { 89 case mask[0] == 254 && mask[1] == 255: 90 fallthrough 91 case mask[0] == 255 && mask[1] == 254: 92 if _, err = p.buf.Read(mask); err != nil { 93 return err 94 } 95 case mask[0] == 239 && mask[1] == 187: 96 mask, err := p.buf.Peek(3) 97 if err != nil && err != io.EOF { 98 return err 99 } else if len(mask) < 3 { 100 return nil 101 } 102 if mask[2] != 191 { 103 return nil 104 } 105 if _, err = p.buf.Read(mask); err != nil { 106 return err 107 } 108 default: 109 return nil 110 } 111 } 112 } 113 114 func (p *parser) readUntil(delim byte) ([]byte, error) { 115 data, err := p.buf.ReadBytes(delim) 116 if err != nil { 117 if err == io.EOF { 118 p.isEOF = true 119 } else { 120 return nil, err 121 } 122 } 123 return data, nil 124 } 125 126 func cleanComment(in []byte) ([]byte, bool) { 127 i := bytes.IndexAny(in, "#;") 128 if i == -1 { 129 return nil, false 130 } 131 return in[i:], true 132 } 133 134 func readKeyName(delimiters string, in []byte) (string, int, error) { 135 line := string(in) 136 137 // Check if key name surrounded by quotes. 138 var keyQuote string 139 switch line[0] { 140 case '"': 141 if len(line) > 6 && line[0:3] == `"""` { 142 keyQuote = `"""` 143 } else { 144 keyQuote = `"` 145 } 146 case '`': 147 keyQuote = "`" 148 } 149 150 // Get out key name 151 var endIdx int 152 if len(keyQuote) > 0 { 153 startIdx := len(keyQuote) 154 // FIXME: fail case -> """"""name"""=value 155 pos := strings.Index(line[startIdx:], keyQuote) 156 if pos == -1 { 157 return "", -1, fmt.Errorf("missing closing key quote: %s", line) 158 } 159 pos += startIdx 160 161 // Find key-value delimiter 162 i := strings.IndexAny(line[pos+startIdx:], delimiters) 163 if i < 0 { 164 return "", -1, ErrDelimiterNotFound{line} 165 } 166 endIdx = pos + i 167 return strings.TrimSpace(line[startIdx:pos]), endIdx + startIdx + 1, nil 168 } 169 170 endIdx = strings.IndexAny(line, delimiters) 171 if endIdx < 0 { 172 return "", -1, ErrDelimiterNotFound{line} 173 } 174 if endIdx == 0 { 175 return "", -1, ErrEmptyKeyName{line} 176 } 177 178 return strings.TrimSpace(line[0:endIdx]), endIdx + 1, nil 179 } 180 181 func (p *parser) readMultilines(line, val, valQuote string) (string, error) { 182 for { 183 data, err := p.readUntil('\n') 184 if err != nil { 185 return "", err 186 } 187 next := string(data) 188 189 pos := strings.LastIndex(next, valQuote) 190 if pos > -1 { 191 // Check if the line ends with backslash continuation after the quote 192 restOfLine := strings.TrimRight(next[pos+len(valQuote):], "\r\n") 193 if !p.options.IgnoreContinuation && strings.HasSuffix(strings.TrimSpace(restOfLine), `\`) { 194 val += next 195 continue 196 } 197 198 val += next[:pos] 199 200 comment, has := cleanComment([]byte(next[pos:])) 201 if has { 202 p.comment.Write(bytes.TrimSpace(comment)) 203 } 204 break 205 } 206 val += next 207 if p.isEOF { 208 return "", fmt.Errorf("missing closing key quote from %q to %q", line, next) 209 } 210 } 211 return val, nil 212 } 213 214 func (p *parser) readContinuationLines(val string) (string, error) { 215 for { 216 data, err := p.readUntil('\n') 217 if err != nil { 218 return "", err 219 } 220 next := strings.TrimSpace(string(data)) 221 222 if len(next) == 0 { 223 break 224 } 225 val += next 226 if val[len(val)-1] != '\\' { 227 break 228 } 229 val = val[:len(val)-1] 230 } 231 return val, nil 232 } 233 234 func inlineCommentIndex(line string, spaceBefore bool) int { 235 var quote byte 236 escaped := false 237 for i := 0; i < len(line); i++ { 238 ch := line[i] 239 if quote != 0 { 240 if escaped { 241 escaped = false 242 continue 243 } 244 if quote == '"' && ch == '\\' { 245 escaped = true 246 continue 247 } 248 if ch == quote { 249 quote = 0 250 } 251 continue 252 } 253 if i == 0 && (ch == '\'' || ch == '"') { 254 quote = ch 255 continue 256 } 257 if ch != '#' && ch != ';' { 258 continue 259 } 260 if !spaceBefore { 261 return i 262 } 263 if i > 0 && line[i-1] == ' ' { 264 return i - 1 265 } 266 } 267 return -1 268 } 269 270 // hasSurroundedQuote check if and only if the first and last characters 271 // are quotes \" or \'. 272 // It returns false if any other parts also contain same kind of quotes. 273 func hasSurroundedQuote(in string, quote byte) bool { 274 return len(in) >= 2 && in[0] == quote && in[len(in)-1] == quote && 275 strings.IndexByte(in[1:], quote) == len(in)-2 276 } 277 278 func (p *parser) readValue(in []byte, bufferSize int) (string, error) { 279 280 line := strings.TrimLeftFunc(string(in), unicode.IsSpace) 281 if len(line) == 0 { 282 if p.options.AllowPythonMultilineValues && len(in) > 0 && in[len(in)-1] == '\n' { 283 return p.readPythonMultilines(line, bufferSize) 284 } 285 return "", nil 286 } 287 288 var valQuote string 289 if len(line) > 3 && line[0:3] == `"""` { 290 valQuote = `"""` 291 } else if line[0] == '`' { 292 valQuote = "`" 293 } else if p.options.UnescapeValueDoubleQuotes && line[0] == '"' { 294 valQuote = `"` 295 } 296 297 if len(valQuote) > 0 { 298 startIdx := len(valQuote) 299 pos := strings.LastIndex(line[startIdx:], valQuote) 300 // Check for multi-line value 301 if pos == -1 { 302 return p.readMultilines(line, line[startIdx:], valQuote) 303 } 304 305 if p.options.UnescapeValueDoubleQuotes && valQuote == `"` { 306 return strings.ReplaceAll(line[startIdx:pos+startIdx], `\"`, `"`), nil 307 } 308 return line[startIdx : pos+startIdx], nil 309 } 310 311 lastChar := line[len(line)-1] 312 // Won't be able to reach here if value only contains whitespace 313 line = strings.TrimSpace(line) 314 trimmedLastChar := line[len(line)-1] 315 316 // Check continuation lines when desired 317 if !p.options.IgnoreContinuation && trimmedLastChar == '\\' { 318 return p.readContinuationLines(line[:len(line)-1]) 319 } 320 321 // Check for inline comments only outside a value's surrounding quotes. 322 if !p.options.IgnoreInlineComment { 323 i := inlineCommentIndex(line, p.options.SpaceBeforeInlineComment) 324 if i > -1 { 325 p.comment.WriteString(line[i:]) 326 line = strings.TrimSpace(line[:i]) 327 } 328 } 329 330 // Trim single and double quotes 331 if (hasSurroundedQuote(line, '\'') || 332 hasSurroundedQuote(line, '"')) && !p.options.PreserveSurroundedQuote { 333 line = line[1 : len(line)-1] 334 } else if len(valQuote) == 0 && p.options.UnescapeValueCommentSymbols { 335 line = strings.ReplaceAll(line, `\;`, ";") 336 line = strings.ReplaceAll(line, `\#`, "#") 337 } else if p.options.AllowPythonMultilineValues && lastChar == '\n' { 338 return p.readPythonMultilines(line, bufferSize) 339 } 340 341 return line, nil 342 } 343 344 func (p *parser) readPythonMultilines(line string, bufferSize int) (string, error) { 345 parserBufferPeekResult, _ := p.buf.Peek(bufferSize) 346 peekBuffer := bytes.NewBuffer(parserBufferPeekResult) 347 348 for { 349 peekData, peekErr := peekBuffer.ReadBytes('\n') 350 if peekErr != nil && peekErr != io.EOF { 351 p.debug("readPythonMultilines: failed to peek with error: %v", peekErr) 352 return "", peekErr 353 } 354 355 p.debug("readPythonMultilines: parsing %q", string(peekData)) 356 357 peekMatches := pythonMultiline.FindStringSubmatch(string(peekData)) 358 p.debug("readPythonMultilines: matched %d parts", len(peekMatches)) 359 for n, v := range peekMatches { 360 p.debug(" %d: %q", n, v) 361 } 362 363 // Return if not a Python multiline value. 364 if len(peekMatches) != 3 { 365 p.debug("readPythonMultilines: end of value, got: %q", line) 366 return line, nil 367 } 368 369 // Advance the parser reader (buffer) in-sync with the peek buffer. 370 _, err := p.buf.Discard(len(peekData)) 371 if err != nil { 372 p.debug("readPythonMultilines: failed to skip to the end, returning error") 373 return "", err 374 } 375 376 line += "\n" + peekMatches[0] 377 } 378 } 379 380 // parse parses data through an io.Reader. 381 func (f *File) parse(reader io.Reader) (err error) { 382 p := newParser(reader, parserOptions{ 383 IgnoreContinuation: f.options.IgnoreContinuation, 384 IgnoreInlineComment: f.options.IgnoreInlineComment, 385 AllowPythonMultilineValues: f.options.AllowPythonMultilineValues, 386 SpaceBeforeInlineComment: f.options.SpaceBeforeInlineComment, 387 UnescapeValueDoubleQuotes: f.options.UnescapeValueDoubleQuotes, 388 UnescapeValueCommentSymbols: f.options.UnescapeValueCommentSymbols, 389 PreserveSurroundedQuote: f.options.PreserveSurroundedQuote, 390 DebugFunc: f.options.DebugFunc, 391 ReaderBufferSize: f.options.ReaderBufferSize, 392 }) 393 if err = p.BOM(); err != nil { 394 return fmt.Errorf("BOM: %v", err) 395 } 396 397 // Ignore error because default section name is never empty string. 398 name := DefaultSection 399 if f.options.Insensitive || f.options.InsensitiveSections { 400 name = strings.ToLower(DefaultSection) 401 } 402 section, _ := f.NewSection(name) 403 404 // This "last" is not strictly equivalent to "previous one" if current key is not the first nested key 405 var isLastValueEmpty bool 406 var lastRegularKey *Key 407 408 var line []byte 409 var inUnparseableSection bool 410 411 // NOTE: Iterate and increase `currentPeekSize` until 412 // the size of the parser buffer is found. 413 // TODO(unknwon): When Golang 1.10 is the lowest version supported, replace with `parserBufferSize := p.buf.Size()`. 414 parserBufferSize := 0 415 // NOTE: Peek 4kb at a time. 416 currentPeekSize := minReaderBufferSize 417 418 if f.options.AllowPythonMultilineValues { 419 for { 420 peekBytes, _ := p.buf.Peek(currentPeekSize) 421 peekBytesLength := len(peekBytes) 422 423 if parserBufferSize >= peekBytesLength { 424 break 425 } 426 427 currentPeekSize *= 2 428 parserBufferSize = peekBytesLength 429 } 430 } 431 432 for !p.isEOF { 433 line, err = p.readUntil('\n') 434 if err != nil { 435 return err 436 } 437 438 if f.options.AllowNestedValues && 439 isLastValueEmpty && len(line) > 0 { 440 if line[0] == ' ' || line[0] == '\t' { 441 err = lastRegularKey.addNestedValue(string(bytes.TrimSpace(line))) 442 if err != nil { 443 return err 444 } 445 continue 446 } 447 } 448 449 line = bytes.TrimLeftFunc(line, unicode.IsSpace) 450 if len(line) == 0 { 451 continue 452 } 453 454 // Comments 455 if line[0] == '#' || line[0] == ';' { 456 // Note: we do not care ending line break, 457 // it is needed for adding second line, 458 // so just clean it once at the end when set to value. 459 p.comment.Write(line) 460 continue 461 } 462 463 // Section 464 if line[0] == '[' { 465 // Read to the next ']' (TODO: support quoted strings) 466 closeIdx := bytes.LastIndexByte(line, ']') 467 if closeIdx == -1 { 468 return fmt.Errorf("unclosed section: %s", line) 469 } 470 471 name := string(line[1:closeIdx]) 472 section, err = f.NewSection(name) 473 if err != nil { 474 return err 475 } 476 477 comment, has := cleanComment(line[closeIdx+1:]) 478 if has { 479 p.comment.Write(comment) 480 } 481 482 section.Comment = strings.TrimSpace(p.comment.String()) 483 484 // Reset auto-counter and comments 485 p.comment.Reset() 486 p.count = 1 487 // Nested values can't span sections 488 isLastValueEmpty = false 489 490 inUnparseableSection = false 491 for i := range f.options.UnparseableSections { 492 if f.options.UnparseableSections[i] == name || 493 ((f.options.Insensitive || f.options.InsensitiveSections) && strings.EqualFold(f.options.UnparseableSections[i], name)) { 494 inUnparseableSection = true 495 section.isRawSection = true 496 continue 497 } 498 } 499 continue 500 } 501 502 if inUnparseableSection { 503 section.rawBody += string(line) 504 continue 505 } 506 507 kname, offset, err := readKeyName(f.options.KeyValueDelimiters, line) 508 if err != nil { 509 switch { 510 // Treat as boolean key when desired, and whole line is key name. 511 case IsErrDelimiterNotFound(err): 512 switch { 513 case f.options.AllowBooleanKeys: 514 kname, err := p.readValue(line, parserBufferSize) 515 if err != nil { 516 return err 517 } 518 key, err := section.NewBooleanKey(kname) 519 if err != nil { 520 return err 521 } 522 key.Comment = strings.TrimSpace(p.comment.String()) 523 p.comment.Reset() 524 continue 525 526 case f.options.SkipUnrecognizableLines: 527 continue 528 } 529 case IsErrEmptyKeyName(err) && f.options.SkipUnrecognizableLines: 530 continue 531 } 532 return err 533 } 534 535 // Auto increment. 536 isAutoIncr := false 537 if kname == "-" { 538 isAutoIncr = true 539 kname = "#" + strconv.Itoa(p.count) 540 p.count++ 541 } 542 543 value, err := p.readValue(line[offset:], parserBufferSize) 544 if err != nil { 545 return err 546 } 547 isLastValueEmpty = len(value) == 0 548 549 key, err := section.NewKey(kname, value) 550 if err != nil { 551 return err 552 } 553 key.isAutoIncrement = isAutoIncr 554 key.Comment = strings.TrimSpace(p.comment.String()) 555 p.comment.Reset() 556 lastRegularKey = key 557 } 558 // Comments after the last key/section are otherwise dropped on EOF 559 // (e.g. Ansible blockinfile # END markers). Keep them on the last section. 560 if p.comment.Len() > 0 { 561 section.TrailingComment = strings.TrimSpace(p.comment.String()) 562 p.comment.Reset() 563 } 564 return nil 565 }