diasks2/pragmatic_tokenizer · tokenizer.rb
ruby logo
def process_numbers!
        case @numbers
        when :semi
          @tokens.delete_if { |token| token =~ Regex::ONLY_DECIMALS }
        when :none
          @tokens.delete_if { |token| token =~ Regex::NO_DECIMALS_NO_NUMERALS }
        when :only
          @tokens.delete_if { |token| token =~ Regex::NO_DECIMALS }
        end
      end
Similar code snippets
1.
diasks2/pragmatic_tokenizer · tokenizer.rb
Match rating: 58.13% · See similar code snippets
ruby logo
def hashtags!
        case @hashtags
        when :remove
          @tokens.delete_if { |token| token =~ Regex::ONLY_HASHTAG }
        when :keep_and_clean
          @tokens.map!      { |token| token =~ Regex::ONLY_HASHTAG ? token[1..-1] : token }
        end
      end
2.
diasks2/pragmatic_tokenizer · tokenizer.rb
Match rating: 56.54% · See similar code snippets
ruby logo
def mentions!
        case @mentions
        when :remove
          @tokens.delete_if { |token| token =~ Regex::ONLY_MENTION }
        when :keep_and_clean
          @tokens.map!      { |token| token =~ Regex::ONLY_MENTION ? token[1..-1] : token }
        end
      end
3.
diasks2/pragmatic_tokenizer · tokenizer.rb
Match rating: 53.23% · See similar code snippets
ruby logo
def process_punctuation!
        case @punctuation
        when :semi
          @tokens.delete_if { |token| token =~ Regex::PUNCTUATION4 }
        when :none
          @tokens.delete_if { |token| token =~ Regex::ONLY_PUNCTUATION }
        when :only
          @tokens.keep_if   { |token| token =~ Regex::ONLY_PUNCTUATION }
        end
      end
4.
kkikzk/src_lexer · src_lexer.rb
Match rating: 52.16% · See similar code snippets
ruby logo
def pop_token
      token = @tokens.shift
      return END_TOKEN if token.nil?
      case token[0]
      when NUMBER_REGEX
        [:NUMBER, Token.new(token[0], token[1], token[2])]
      when STRING_REGEX
        [:STRING, Token.new(token[0], token[1], token[2])]
      else
        [is_reserved?(token[0]) ? token[0] : :IDENT, Token.new(token[0], token[1], token[2])]
      end
    end
5.
pazdera/tco · parser.rb
Match rating: 52.01% · See similar code snippets
ruby logo
def submit_token(type, value)
      return if value == ""

      if type == :normal && @tokens.length > 0 && @tokens[-1].type == :normal
        prev = @tokens.pop
        @tokens.push Token.new :normal, prev.to_s + value
      else
        @tokens.push Token.new type, value
      end
    end
6.
diasks2/pragmatic_tokenizer · tokenizer.rb
Match rating: 51.36% · See similar code snippets
ruby logo
def clean!
        @tokens = @tokens
            .flat_map  { |token| split_underscores_asterisk(token) }
            .map!      { |token| remove_irrelevant_characters(token) }
            .delete_if { |token| many_dots?(token) }
      end
7.
github/linguist · classifier.rb
Match rating: 51.15% · See similar code snippets
ruby logo
def token_probability(token, language)
      if @tokens[language][token].to_f == 0.0
        1 / @tokens_total.to_f
      else
        @tokens[language][token].to_f / @language_tokens[language].to_f
      end
    end
8.
jonatas/fast · fast.rb
Match rating: 50.18% · See similar code snippets
ruby logo
def parse_until_peek(token)
      list = []
      list << parse until @tokens.empty? || @tokens.first == token
      next_token
      list
    end
9.
cxn03651/writeexcel · worksheet.rb
Match rating: 50.13% · See similar code snippets
ruby logo
def extract_filter_tokens(expression = nil)   #:nodoc:
    return [] unless expression

    tokens = []
    str = expression
    while str =~ /"(?:[^"]|"")*"|\S+/
      tokens << $&
      str = $~.post_match
    end

    # Remove leading and trailing quotes and unescape other quotes
    tokens.map! do |token|
      token.sub!(/^"/, '')
      token.sub!(/"$/, '')
      token.gsub!(/""/, '"')

      # if token is number, convert to numeric.
      if token =~ /^([+-]?)(?=\d|\.\d)\d*(\.\d*)?([Ee]([+-]?\d+))?$/
        token.to_f == token.to_i ? token.to_i : token.to_f
      else
        token
      end
    end

    tokens
  end
10.
inukshuk/bibtex-ruby · value.rb
Match rating: 49.73% · See similar code snippets
ruby logo
def initialize_copy(other)
      @tokens = other.tokens.map do |token|
        case
        when token.nil? then nil
        when token.is_a?(Symbol) then token
        else token.dup
        end
      end
    end