Class: Sass::Util::MultibyteStringScanner

Inherits:
StringScanner
  • Object
show all
Defined in:
/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb,
/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb,
/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb

Overview

A wrapper of the native StringScanner class that works correctly with multibyte character encodings. The native class deals only in bytes, not characters, for methods like [#pos] and [#matched_size]. This class deals only in characters, instead.

Class Method Summary (collapse)

Instance Method Summary (collapse)

Constructor Details

- (MultibyteStringScanner) initialize(str)

Returns a new instance of MultibyteStringScanner



9
10
11
12
13
14
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 9

def initialize(str)
  super(StringScanner.new(str))
  @mb_pos = 0
  @mb_matched_size = nil
  @mb_last_pos = nil
end

Class Method Details

+ new(str)



36
37
38
39
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 36

def self.new(str)
  return StringScanner.new(str) if str.ascii_only?
  super
end

Instance Method Details

- byte_matched_size



42
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 42

alias_method :byte_matched_size, :matched_size

- byte_pos



41
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 41

alias_method :byte_pos, :pos

- check(pattern)



44
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 44

def check(pattern); _match super; end

- check_until(pattern)



45
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 45

def check_until(pattern); _matched super; end

- get_byte



59
60
61
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 59

def get_byte
  raise "MultibyteStringScanner doesn't support #get_byte."
end

- getbyte



63
64
65
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 63

def getbyte
  raise "MultibyteStringScanner doesn't support #getbyte."
end

- getch



46
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 46

def getch; _forward _match super; end

- (Boolean) is_a?(klass)

Returns:

  • (Boolean)


16
17
18
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 16

def is_a?(klass)
  __getobj__.is_a?(klass) || super
end

- (Boolean) match?(pattern)

Returns:

  • (Boolean)


47
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 47

def match?(pattern); _size check(pattern); end

- matched_size



48
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 48

def matched_size; @mb_matched_size; end

- peek(len) Also known as: peep



49
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 49

def peek(len); string[@mb_pos, len]; end

- pos Also known as: pointer



51
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 51

def pos; @mb_pos; end

- pos=(n)



67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 67

def pos=(n)
  @mb_last_pos = nil

  # We set position kind of a lot during parsing, so we want it to be as
  # efficient as possible. This is complicated by the fact that UTF-8 is a
  # variable-length encoding, so it's difficult to find the byte length that
  # corresponds to a given character length.
  #
  # Our heuristic here is to try to count the fewest possible characters. So
  # if the new position is close to the current one, just count the
  # characters between the two; if the new position is closer to the
  # beginning of the string, just count the characters from there.
  if @mb_pos - n < @mb_pos / 2
    # New position is close to old position
    byte_delta = @mb_pos > n ? -string[n...@mb_pos].bytesize : string[@mb_pos...n].bytesize
    super(byte_pos + byte_delta)
  else
    # New position is close to BOS
    super(string[0...n].bytesize)
  end
  @mb_pos = n
end

- reset



90
91
92
93
94
95
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 90

def reset
  @mb_pos = 0
  @mb_matched_size = nil
  @mb_last_pos = nil
  super
end

- rest_size



53
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 53

def rest_size; rest.size; end

- scan(pattern)



54
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 54

def scan(pattern); _forward _match super; end

- scan_full(pattern, advance_pointer_p, return_string_p)



97
98
99
100
101
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 97

def scan_full(pattern, advance_pointer_p, return_string_p)
  res = _match super(pattern, advance_pointer_p, true)
  _forward res if advance_pointer_p
  return res if return_string_p
end

- scan_until(pattern)



55
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 55

def scan_until(pattern); _forward _matched super; end

- search_full(pattern, advance_pointer_p, return_string_p)



103
104
105
106
107
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 103

def search_full(pattern, advance_pointer_p, return_string_p)
  res = super(pattern, advance_pointer_p, true)
  _forward res if advance_pointer_p
  _matched((res if return_string_p))
end

- skip(pattern)



56
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 56

def skip(pattern); _size scan(pattern); end

- skip_until(pattern)



57
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 57

def skip_until(pattern); _matched _size scan_until(pattern); end

- string=(str)



109
110
111
112
113
114
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 109

def string=(str)
  @mb_pos = 0
  @mb_matched_size = nil
  @mb_last_pos = nil
  super
end

- terminate Also known as: clear



116
117
118
119
120
121
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 116

def terminate
  @mb_pos = string.size
  @mb_matched_size = nil
  @mb_last_pos = nil
  super
end

- unscan



124
125
126
127
128
# File '/home/nweiz/code/sass-site/.sass/lib/sass/util/multibyte_string_scanner.rb', line 124

def unscan
  super
  @mb_pos = @mb_last_pos
  @mb_last_pos = @mb_matched_size = nil
end