taler-mailbox

Service for asynchronous wallet-to-wallet payment messages
Log | Files | Refs | Submodules | README | LICENSE

parser.go (14420B)


      1 // Copyright 2015 Unknwon
      2 //
      3 // Licensed under the Apache License, Version 2.0 (the "License"): you may
      4 // not use this file except in compliance with the License. You may obtain
      5 // a copy of the License at
      6 //
      7 //     http://www.apache.org/licenses/LICENSE-2.0
      8 //
      9 // Unless required by applicable law or agreed to in writing, software
     10 // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
     11 // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
     12 // License for the specific language governing permissions and limitations
     13 // under the License.
     14 
     15 package ini
     16 
     17 import (
     18 	"bufio"
     19 	"bytes"
     20 	"fmt"
     21 	"io"
     22 	"regexp"
     23 	"strconv"
     24 	"strings"
     25 	"unicode"
     26 )
     27 
     28 const minReaderBufferSize = 4096
     29 
     30 var pythonMultiline = regexp.MustCompile(`^([\t\f ]+)(.*)`)
     31 
     32 type parserOptions struct {
     33 	IgnoreContinuation          bool
     34 	IgnoreInlineComment         bool
     35 	AllowPythonMultilineValues  bool
     36 	SpaceBeforeInlineComment    bool
     37 	UnescapeValueDoubleQuotes   bool
     38 	UnescapeValueCommentSymbols bool
     39 	PreserveSurroundedQuote     bool
     40 	DebugFunc                   DebugFunc
     41 	ReaderBufferSize            int
     42 }
     43 
     44 type parser struct {
     45 	buf     *bufio.Reader
     46 	options parserOptions
     47 
     48 	isEOF   bool
     49 	count   int
     50 	comment *bytes.Buffer
     51 }
     52 
     53 func (p *parser) debug(format string, args ...interface{}) {
     54 	if p.options.DebugFunc != nil {
     55 		p.options.DebugFunc(fmt.Sprintf(format, args...))
     56 	}
     57 }
     58 
     59 func newParser(r io.Reader, opts parserOptions) *parser {
     60 	size := opts.ReaderBufferSize
     61 	if size < minReaderBufferSize {
     62 		size = minReaderBufferSize
     63 	}
     64 
     65 	return &parser{
     66 		buf:     bufio.NewReaderSize(r, size),
     67 		options: opts,
     68 		count:   1,
     69 		comment: &bytes.Buffer{},
     70 	}
     71 }
     72 
     73 // BOM handles header of UTF-8, UTF-16 LE and UTF-16 BE's BOM format.
     74 // http://en.wikipedia.org/wiki/Byte_order_mark#Representations_of_byte_order_marks_by_encoding
     75 func (p *parser) BOM() error {
     76 	// Strip every leading BOM: a file may accidentally carry more than one
     77 	// (e.g. a tool prepending a BOM to already-BOM'd content). Stopping after
     78 	// the first one leaves the remaining BOM on the next token and makes the
     79 	// parser fail with a misleading "key-value delimiter not found" (#390).
     80 	for {
     81 		mask, err := p.buf.Peek(2)
     82 		if err != nil && err != io.EOF {
     83 			return err
     84 		} else if len(mask) < 2 {
     85 			return nil
     86 		}
     87 
     88 		switch {
     89 		case mask[0] == 254 && mask[1] == 255:
     90 			fallthrough
     91 		case mask[0] == 255 && mask[1] == 254:
     92 			if _, err = p.buf.Read(mask); err != nil {
     93 				return err
     94 			}
     95 		case mask[0] == 239 && mask[1] == 187:
     96 			mask, err := p.buf.Peek(3)
     97 			if err != nil && err != io.EOF {
     98 				return err
     99 			} else if len(mask) < 3 {
    100 				return nil
    101 			}
    102 			if mask[2] != 191 {
    103 				return nil
    104 			}
    105 			if _, err = p.buf.Read(mask); err != nil {
    106 				return err
    107 			}
    108 		default:
    109 			return nil
    110 		}
    111 	}
    112 }
    113 
    114 func (p *parser) readUntil(delim byte) ([]byte, error) {
    115 	data, err := p.buf.ReadBytes(delim)
    116 	if err != nil {
    117 		if err == io.EOF {
    118 			p.isEOF = true
    119 		} else {
    120 			return nil, err
    121 		}
    122 	}
    123 	return data, nil
    124 }
    125 
    126 func cleanComment(in []byte) ([]byte, bool) {
    127 	i := bytes.IndexAny(in, "#;")
    128 	if i == -1 {
    129 		return nil, false
    130 	}
    131 	return in[i:], true
    132 }
    133 
    134 func readKeyName(delimiters string, in []byte) (string, int, error) {
    135 	line := string(in)
    136 
    137 	// Check if key name surrounded by quotes.
    138 	var keyQuote string
    139 	switch line[0] {
    140 	case '"':
    141 		if len(line) > 6 && line[0:3] == `"""` {
    142 			keyQuote = `"""`
    143 		} else {
    144 			keyQuote = `"`
    145 		}
    146 	case '`':
    147 		keyQuote = "`"
    148 	}
    149 
    150 	// Get out key name
    151 	var endIdx int
    152 	if len(keyQuote) > 0 {
    153 		startIdx := len(keyQuote)
    154 		// FIXME: fail case -> """"""name"""=value
    155 		pos := strings.Index(line[startIdx:], keyQuote)
    156 		if pos == -1 {
    157 			return "", -1, fmt.Errorf("missing closing key quote: %s", line)
    158 		}
    159 		pos += startIdx
    160 
    161 		// Find key-value delimiter
    162 		i := strings.IndexAny(line[pos+startIdx:], delimiters)
    163 		if i < 0 {
    164 			return "", -1, ErrDelimiterNotFound{line}
    165 		}
    166 		endIdx = pos + i
    167 		return strings.TrimSpace(line[startIdx:pos]), endIdx + startIdx + 1, nil
    168 	}
    169 
    170 	endIdx = strings.IndexAny(line, delimiters)
    171 	if endIdx < 0 {
    172 		return "", -1, ErrDelimiterNotFound{line}
    173 	}
    174 	if endIdx == 0 {
    175 		return "", -1, ErrEmptyKeyName{line}
    176 	}
    177 
    178 	return strings.TrimSpace(line[0:endIdx]), endIdx + 1, nil
    179 }
    180 
    181 func (p *parser) readMultilines(line, val, valQuote string) (string, error) {
    182 	for {
    183 		data, err := p.readUntil('\n')
    184 		if err != nil {
    185 			return "", err
    186 		}
    187 		next := string(data)
    188 
    189 		pos := strings.LastIndex(next, valQuote)
    190 		if pos > -1 {
    191 			// Check if the line ends with backslash continuation after the quote
    192 			restOfLine := strings.TrimRight(next[pos+len(valQuote):], "\r\n")
    193 			if !p.options.IgnoreContinuation && strings.HasSuffix(strings.TrimSpace(restOfLine), `\`) {
    194 				val += next
    195 				continue
    196 			}
    197 
    198 			val += next[:pos]
    199 
    200 			comment, has := cleanComment([]byte(next[pos:]))
    201 			if has {
    202 				p.comment.Write(bytes.TrimSpace(comment))
    203 			}
    204 			break
    205 		}
    206 		val += next
    207 		if p.isEOF {
    208 			return "", fmt.Errorf("missing closing key quote from %q to %q", line, next)
    209 		}
    210 	}
    211 	return val, nil
    212 }
    213 
    214 func (p *parser) readContinuationLines(val string) (string, error) {
    215 	for {
    216 		data, err := p.readUntil('\n')
    217 		if err != nil {
    218 			return "", err
    219 		}
    220 		next := strings.TrimSpace(string(data))
    221 
    222 		if len(next) == 0 {
    223 			break
    224 		}
    225 		val += next
    226 		if val[len(val)-1] != '\\' {
    227 			break
    228 		}
    229 		val = val[:len(val)-1]
    230 	}
    231 	return val, nil
    232 }
    233 
    234 func inlineCommentIndex(line string, spaceBefore bool) int {
    235 	var quote byte
    236 	escaped := false
    237 	for i := 0; i < len(line); i++ {
    238 		ch := line[i]
    239 		if quote != 0 {
    240 			if escaped {
    241 				escaped = false
    242 				continue
    243 			}
    244 			if quote == '"' && ch == '\\' {
    245 				escaped = true
    246 				continue
    247 			}
    248 			if ch == quote {
    249 				quote = 0
    250 			}
    251 			continue
    252 		}
    253 		if i == 0 && (ch == '\'' || ch == '"') {
    254 			quote = ch
    255 			continue
    256 		}
    257 		if ch != '#' && ch != ';' {
    258 			continue
    259 		}
    260 		if !spaceBefore {
    261 			return i
    262 		}
    263 		if i > 0 && line[i-1] == ' ' {
    264 			return i - 1
    265 		}
    266 	}
    267 	return -1
    268 }
    269 
    270 // hasSurroundedQuote check if and only if the first and last characters
    271 // are quotes \" or \'.
    272 // It returns false if any other parts also contain same kind of quotes.
    273 func hasSurroundedQuote(in string, quote byte) bool {
    274 	return len(in) >= 2 && in[0] == quote && in[len(in)-1] == quote &&
    275 		strings.IndexByte(in[1:], quote) == len(in)-2
    276 }
    277 
    278 func (p *parser) readValue(in []byte, bufferSize int) (string, error) {
    279 
    280 	line := strings.TrimLeftFunc(string(in), unicode.IsSpace)
    281 	if len(line) == 0 {
    282 		if p.options.AllowPythonMultilineValues && len(in) > 0 && in[len(in)-1] == '\n' {
    283 			return p.readPythonMultilines(line, bufferSize)
    284 		}
    285 		return "", nil
    286 	}
    287 
    288 	var valQuote string
    289 	if len(line) > 3 && line[0:3] == `"""` {
    290 		valQuote = `"""`
    291 	} else if line[0] == '`' {
    292 		valQuote = "`"
    293 	} else if p.options.UnescapeValueDoubleQuotes && line[0] == '"' {
    294 		valQuote = `"`
    295 	}
    296 
    297 	if len(valQuote) > 0 {
    298 		startIdx := len(valQuote)
    299 		pos := strings.LastIndex(line[startIdx:], valQuote)
    300 		// Check for multi-line value
    301 		if pos == -1 {
    302 			return p.readMultilines(line, line[startIdx:], valQuote)
    303 		}
    304 
    305 		if p.options.UnescapeValueDoubleQuotes && valQuote == `"` {
    306 			return strings.ReplaceAll(line[startIdx:pos+startIdx], `\"`, `"`), nil
    307 		}
    308 		return line[startIdx : pos+startIdx], nil
    309 	}
    310 
    311 	lastChar := line[len(line)-1]
    312 	// Won't be able to reach here if value only contains whitespace
    313 	line = strings.TrimSpace(line)
    314 	trimmedLastChar := line[len(line)-1]
    315 
    316 	// Check continuation lines when desired
    317 	if !p.options.IgnoreContinuation && trimmedLastChar == '\\' {
    318 		return p.readContinuationLines(line[:len(line)-1])
    319 	}
    320 
    321 	// Check for inline comments only outside a value's surrounding quotes.
    322 	if !p.options.IgnoreInlineComment {
    323 		i := inlineCommentIndex(line, p.options.SpaceBeforeInlineComment)
    324 		if i > -1 {
    325 			p.comment.WriteString(line[i:])
    326 			line = strings.TrimSpace(line[:i])
    327 		}
    328 	}
    329 
    330 	// Trim single and double quotes
    331 	if (hasSurroundedQuote(line, '\'') ||
    332 		hasSurroundedQuote(line, '"')) && !p.options.PreserveSurroundedQuote {
    333 		line = line[1 : len(line)-1]
    334 	} else if len(valQuote) == 0 && p.options.UnescapeValueCommentSymbols {
    335 		line = strings.ReplaceAll(line, `\;`, ";")
    336 		line = strings.ReplaceAll(line, `\#`, "#")
    337 	} else if p.options.AllowPythonMultilineValues && lastChar == '\n' {
    338 		return p.readPythonMultilines(line, bufferSize)
    339 	}
    340 
    341 	return line, nil
    342 }
    343 
    344 func (p *parser) readPythonMultilines(line string, bufferSize int) (string, error) {
    345 	parserBufferPeekResult, _ := p.buf.Peek(bufferSize)
    346 	peekBuffer := bytes.NewBuffer(parserBufferPeekResult)
    347 
    348 	for {
    349 		peekData, peekErr := peekBuffer.ReadBytes('\n')
    350 		if peekErr != nil && peekErr != io.EOF {
    351 			p.debug("readPythonMultilines: failed to peek with error: %v", peekErr)
    352 			return "", peekErr
    353 		}
    354 
    355 		p.debug("readPythonMultilines: parsing %q", string(peekData))
    356 
    357 		peekMatches := pythonMultiline.FindStringSubmatch(string(peekData))
    358 		p.debug("readPythonMultilines: matched %d parts", len(peekMatches))
    359 		for n, v := range peekMatches {
    360 			p.debug("   %d: %q", n, v)
    361 		}
    362 
    363 		// Return if not a Python multiline value.
    364 		if len(peekMatches) != 3 {
    365 			p.debug("readPythonMultilines: end of value, got: %q", line)
    366 			return line, nil
    367 		}
    368 
    369 		// Advance the parser reader (buffer) in-sync with the peek buffer.
    370 		_, err := p.buf.Discard(len(peekData))
    371 		if err != nil {
    372 			p.debug("readPythonMultilines: failed to skip to the end, returning error")
    373 			return "", err
    374 		}
    375 
    376 		line += "\n" + peekMatches[0]
    377 	}
    378 }
    379 
    380 // parse parses data through an io.Reader.
    381 func (f *File) parse(reader io.Reader) (err error) {
    382 	p := newParser(reader, parserOptions{
    383 		IgnoreContinuation:          f.options.IgnoreContinuation,
    384 		IgnoreInlineComment:         f.options.IgnoreInlineComment,
    385 		AllowPythonMultilineValues:  f.options.AllowPythonMultilineValues,
    386 		SpaceBeforeInlineComment:    f.options.SpaceBeforeInlineComment,
    387 		UnescapeValueDoubleQuotes:   f.options.UnescapeValueDoubleQuotes,
    388 		UnescapeValueCommentSymbols: f.options.UnescapeValueCommentSymbols,
    389 		PreserveSurroundedQuote:     f.options.PreserveSurroundedQuote,
    390 		DebugFunc:                   f.options.DebugFunc,
    391 		ReaderBufferSize:            f.options.ReaderBufferSize,
    392 	})
    393 	if err = p.BOM(); err != nil {
    394 		return fmt.Errorf("BOM: %v", err)
    395 	}
    396 
    397 	// Ignore error because default section name is never empty string.
    398 	name := DefaultSection
    399 	if f.options.Insensitive || f.options.InsensitiveSections {
    400 		name = strings.ToLower(DefaultSection)
    401 	}
    402 	section, _ := f.NewSection(name)
    403 
    404 	// This "last" is not strictly equivalent to "previous one" if current key is not the first nested key
    405 	var isLastValueEmpty bool
    406 	var lastRegularKey *Key
    407 
    408 	var line []byte
    409 	var inUnparseableSection bool
    410 
    411 	// NOTE: Iterate and increase `currentPeekSize` until
    412 	// the size of the parser buffer is found.
    413 	// TODO(unknwon): When Golang 1.10 is the lowest version supported, replace with `parserBufferSize := p.buf.Size()`.
    414 	parserBufferSize := 0
    415 	// NOTE: Peek 4kb at a time.
    416 	currentPeekSize := minReaderBufferSize
    417 
    418 	if f.options.AllowPythonMultilineValues {
    419 		for {
    420 			peekBytes, _ := p.buf.Peek(currentPeekSize)
    421 			peekBytesLength := len(peekBytes)
    422 
    423 			if parserBufferSize >= peekBytesLength {
    424 				break
    425 			}
    426 
    427 			currentPeekSize *= 2
    428 			parserBufferSize = peekBytesLength
    429 		}
    430 	}
    431 
    432 	for !p.isEOF {
    433 		line, err = p.readUntil('\n')
    434 		if err != nil {
    435 			return err
    436 		}
    437 
    438 		if f.options.AllowNestedValues &&
    439 			isLastValueEmpty && len(line) > 0 {
    440 			if line[0] == ' ' || line[0] == '\t' {
    441 				err = lastRegularKey.addNestedValue(string(bytes.TrimSpace(line)))
    442 				if err != nil {
    443 					return err
    444 				}
    445 				continue
    446 			}
    447 		}
    448 
    449 		line = bytes.TrimLeftFunc(line, unicode.IsSpace)
    450 		if len(line) == 0 {
    451 			continue
    452 		}
    453 
    454 		// Comments
    455 		if line[0] == '#' || line[0] == ';' {
    456 			// Note: we do not care ending line break,
    457 			// it is needed for adding second line,
    458 			// so just clean it once at the end when set to value.
    459 			p.comment.Write(line)
    460 			continue
    461 		}
    462 
    463 		// Section
    464 		if line[0] == '[' {
    465 			// Read to the next ']' (TODO: support quoted strings)
    466 			closeIdx := bytes.LastIndexByte(line, ']')
    467 			if closeIdx == -1 {
    468 				return fmt.Errorf("unclosed section: %s", line)
    469 			}
    470 
    471 			name := string(line[1:closeIdx])
    472 			section, err = f.NewSection(name)
    473 			if err != nil {
    474 				return err
    475 			}
    476 
    477 			comment, has := cleanComment(line[closeIdx+1:])
    478 			if has {
    479 				p.comment.Write(comment)
    480 			}
    481 
    482 			section.Comment = strings.TrimSpace(p.comment.String())
    483 
    484 			// Reset auto-counter and comments
    485 			p.comment.Reset()
    486 			p.count = 1
    487 			// Nested values can't span sections
    488 			isLastValueEmpty = false
    489 
    490 			inUnparseableSection = false
    491 			for i := range f.options.UnparseableSections {
    492 				if f.options.UnparseableSections[i] == name ||
    493 					((f.options.Insensitive || f.options.InsensitiveSections) && strings.EqualFold(f.options.UnparseableSections[i], name)) {
    494 					inUnparseableSection = true
    495 					section.isRawSection = true
    496 					continue
    497 				}
    498 			}
    499 			continue
    500 		}
    501 
    502 		if inUnparseableSection {
    503 			section.rawBody += string(line)
    504 			continue
    505 		}
    506 
    507 		kname, offset, err := readKeyName(f.options.KeyValueDelimiters, line)
    508 		if err != nil {
    509 			switch {
    510 			// Treat as boolean key when desired, and whole line is key name.
    511 			case IsErrDelimiterNotFound(err):
    512 				switch {
    513 				case f.options.AllowBooleanKeys:
    514 					kname, err := p.readValue(line, parserBufferSize)
    515 					if err != nil {
    516 						return err
    517 					}
    518 					key, err := section.NewBooleanKey(kname)
    519 					if err != nil {
    520 						return err
    521 					}
    522 					key.Comment = strings.TrimSpace(p.comment.String())
    523 					p.comment.Reset()
    524 					continue
    525 
    526 				case f.options.SkipUnrecognizableLines:
    527 					continue
    528 				}
    529 			case IsErrEmptyKeyName(err) && f.options.SkipUnrecognizableLines:
    530 				continue
    531 			}
    532 			return err
    533 		}
    534 
    535 		// Auto increment.
    536 		isAutoIncr := false
    537 		if kname == "-" {
    538 			isAutoIncr = true
    539 			kname = "#" + strconv.Itoa(p.count)
    540 			p.count++
    541 		}
    542 
    543 		value, err := p.readValue(line[offset:], parserBufferSize)
    544 		if err != nil {
    545 			return err
    546 		}
    547 		isLastValueEmpty = len(value) == 0
    548 
    549 		key, err := section.NewKey(kname, value)
    550 		if err != nil {
    551 			return err
    552 		}
    553 		key.isAutoIncrement = isAutoIncr
    554 		key.Comment = strings.TrimSpace(p.comment.String())
    555 		p.comment.Reset()
    556 		lastRegularKey = key
    557 	}
    558 	// Comments after the last key/section are otherwise dropped on EOF
    559 	// (e.g. Ansible blockinfile # END markers). Keep them on the last section.
    560 	if p.comment.Len() > 0 {
    561 		section.TrailingComment = strings.TrimSpace(p.comment.String())
    562 		p.comment.Reset()
    563 	}
    564 	return nil
    565 }