diasks2/pragmatic_tokenizer · post_processor.rb
ruby logo
def call
      text
          .split
          .map      { |token| convert_sym_to_punct(token) }
          .flat_map { |token| token.split(Regex::COMMAS_OR_PUNCTUATION) }
          .flat_map { |token| token.split(Regex::VARIOUS) }
          .flat_map { |token| token.split(Regex::ENDS_WITH_PUNCTUATION2) }
          .flat_map { |token| split_dotted_email_or_digit(token) }
          .flat_map { |token| split_abbreviations(token) }
          .flat_map { |token| split_period_after_last_word(token) }
    end
Similar code snippets
1.
manorie/textoken · tokenizer.rb
Match rating: 47.83% · See similar code snippets
ruby logo
def split_punctuations
      col.each do |w|
        @result += Scanner.new(w, default_regexp).result || [w]
      end
    end
2.
conanite/nydp · parser.rb
Match rating: 47.52% · See similar code snippets
ruby logo
def read_list token_stream, termination_token, list=[]
      token = token_stream.next_token
      while token != nil && token.first != termination_token
        list << next_form(token, token_stream)
        token = token_stream.next_token
      end
      Pair.parse_list list
    end
3.
koara/koara-rb · parser.rb
Match rating: 45.49% · See similar code snippets
ruby logo
def line_break
      linebreak = Ast::LineBreak.new
      @tree.open_scope
      while get_next_token_kind == TokenManager::SPACE || get_next_token_kind == TokenManager::TAB
        consume_token(get_next_token_kind)
      end
      token = consume_token(TokenManager::EOL)
      linebreak.explicit = token.image.start_with?("  ");
      @tree.close_scope(linebreak)
    end
4.
michaeledgar/laser · lexical_analysis.rb
Match rating: 43.65% · See similar code snippets
ruby logo
def lex(body = self.body, token_class = Token)
      return [] if body =~ /^#.*encoding.*/
      Ripper.lex(body).map {|token| token_class.new(token) }
    end
5.
zweilove/css_splitter · application_helper.rb
Match rating: 43.38% · See similar code snippets
ruby logo
def split_stylesheet_link_tag(*sources)
      options     = sources.extract_options!
      split_count = options.delete(:split_count) || 2

      sources.map do |source|
        split_sources = (2..split_count).map { |index| "#{source}_split#{index}" }
        split_options = options.dup
        if Rails.env == 'development' && !split_options.key?(:debug)
          split_options[:debug] = false
        end
        split_sources << split_options

        [
          stylesheet_link_tag(source, options),
          "<!--[if lte IE 9]>",
          stylesheet_link_tag(*split_sources),
          "<![endif]-->"
        ]
      end.flatten.join("\n").html_safe
    end
6.
penseo/firefighter · identitytoolkit.rb
Match rating: 42.97% · See similar code snippets
ruby logo
def paginate(url, method: :post, headers: {}, page_size: 100, max_iterations: 1000)
      next_page_token = nil

      max_iterations.times do
        data = {
          nextPageToken: next_page_token,
          maxResults: page_size,
        }
        data = call(method, url, data, headers)

        yield data

        return if data['nextPageToken']&.empty? || data['nextPageToken'] == next_page_token

        next_page_token = data['nextPageToken']
      end
    end
7.
abitdodgy/words_counted · tokeniser.rb
Match rating: 41.76% · See similar code snippets
ruby logo
def tokenise(pattern: TOKEN_REGEXP, exclude: nil)
      filter_proc = filter_to_proc(exclude)
      @input.scan(pattern).map(&:downcase).reject { |token| filter_proc.call(token) }
    end
8.
Match rating: 41.5% · See similar code snippets
ruby logo
def values
      value.last.map do |val|
        calling_mapper.for([value.first, val]).result
      end.flat_map(&:last)
    end
9.
kontena/kontena · remove_command.rb
Match rating: 40.05% · See similar code snippets
ruby logo
def execute
      require_api_url
      token = require_token
      confirm unless forced?

      email_list.each do |email|
        begin
          client(token).delete("users/#{email}")
        rescue => ex
          $stderr.puts pastel.red("Failed to remove user #{email} : #{ex.message}")
        end
      end
    end
10.
jamesdabbs/rspec-coverage · result.rb
Match rating: 39.77% · See similar code snippets
ruby logo
def traverse! *results
      results.unshift self
      files = results.map(&:keys).flatten.uniq
      files.each do |file|
        next unless file.start_with? root

        lines = results.map{ |r| r[file] || [] }
        len   = lines.map(&:length).max

        @coverage[file] = len.times.map do |i|
          yield file, *lines.map { |ls| ls[i] }
        end
      end
      self
    end