class YARD::Parser::SourceParser

Registers a callback to be called after an individual file is parsed. The block passed to this method will be called on subsequent parse calls.

To register a callback that is called after the entire list of files is processed, see {after_parse_list}.

@example Printing the length of each file after it is parsed

SourceParser.after_parse_file do |parser|
  puts "#{parser.file} is #{parser.contents.size} characters"
end
YARD.parse('lib   /*.rb')
# prints:
"lib/foo.rb is 1240 characters"
"lib/foo_bar.rb is 248 characters"

@yield [parser] the yielded block is called once after each file

that is parsed. This might happen many times for a single codebase.

@yieldparam [SourceParser] parser the parser object that parsed

the file.

@yieldreturn [void] the return value for the block is ignored. @return [Proc] the yielded block @see ::before_parse_file @see ::after_parse_list @since 0.7.0

# File lib/yard/parser/source_parser.rb, line 323
def after_parse_file(&block)
  after_parse_file_callbacks << block
end

@return [Array<Proc>] the list of callbacks to be called after

parsing a file. Should only be used for testing.

@since 0.7.0

# File lib/yard/parser/source_parser.rb, line 351
def after_parse_file_callbacks
  @after_parse_file_callbacks ||= []
end

Registers a callback to be called after a list of files is parsed via {parse}. The block passed to this method will be called on subsequent parse calls.

@example Printing results after parsing occurs

SourceParser.after_parse_list do
  puts "Finished parsing!"
end
YARD.parse
# Prints "Finished parsing!" after parsing files

@yield [files, globals] the yielded block is called once before

parsing all files

@yieldparam [Array<String>] files the list of files that will be parsed. @yieldparam [OpenStruct] globals a global structure to store arbitrary

state for post processing (see {Handlers::Processor#globals})

@yieldreturn [void] the return value for the block is ignored. @return [Proc] the yielded block @see ::before_parse_list @see ::before_parse_file @since 0.7.0

# File lib/yard/parser/source_parser.rb, line 257
def after_parse_list(&block)
  after_parse_list_callbacks << block
end

@return [Array<Proc>] the list of callbacks to be called after

parsing a list of files. Should only be used for testing.

@since 0.7.0

# File lib/yard/parser/source_parser.rb, line 337
def after_parse_list_callbacks
  @after_parse_list_callbacks ||= []
end

Registers a callback to be called before an individual file is parsed. The block passed to this method will be called on subsequent parse calls.

To register a callback that is called before the entire list of files is processed, see {before_parse_list}.

@example Installing a simple callback

SourceParser.before_parse_file do |parser|
  puts "I'm parsing #{parser.file}"
end
YARD.parse('lib   /*.rb')
# prints:
"I'm parsing lib/foo.rb"
"I'm parsing lib/foo_bar.rb"
"I'm parsing lib/last_file.rb"

@example Cancel parsing of any test_*.rb files

SourceParser.before_parse_file do |parser|
  return false if parser.file =~ /^test_.+\.rb$/
end

@yield [parser] the yielded block is called once before each

file that is parsed. This might happen many times for a single
codebase.

@yieldparam [SourceParser] parser the parser object that will {#parse}

the file.

@yieldreturn [Boolean] if the block returns false, parsing for

the file is cancelled.

@return [Proc] the yielded block @see ::after_parse_file @see ::before_parse_list @since 0.7.0

# File lib/yard/parser/source_parser.rb, line 294
def before_parse_file(&block)
  before_parse_file_callbacks << block
end

@return [Array<Proc>] the list of callbacks to be called before

parsing a file. Should only be used for testing.

@since 0.7.0

# File lib/yard/parser/source_parser.rb, line 344
def before_parse_file_callbacks
  @before_parse_file_callbacks ||= []
end

Registers a callback to be called before a list of files is parsed via {parse}. The block passed to this method will be called on subsequent parse calls.

@example Installing a simple callback

SourceParser.before_parse_list do |files, globals|
  puts "Starting to parse..."
end
YARD.parse('lib   /*.rb')
# prints "Starting to parse..."

@example Setting global state

SourceParser.before_parse_list do |files, globals|
  globals.method_count = 0
end
SourceParser.after_parse_list do |files, globals|
  puts "Found #{globals.method_count} methods"
end
class MyCountHandler < Handlers::Ruby::Base
  handles :def, :defs
  process { globals.method_count += 1 }
end
YARD.parse
# Prints: "Found 37 methods"

@example Using a global callback to cancel parsing

SourceParser.before_parse_list do |files, globals|
  return false if files.include?('foo.rb')
end

YARD.parse(['foo.rb', 'bar.rb']) # callback cancels this method
YARD.parse('bar.rb') # parses normally

@yield [files, globals] the yielded block is called once before

parsing all files

@yieldparam [Array<String>] files the list of files that will be parsed. @yieldparam [OpenStruct] globals a global structure to store arbitrary

state for post processing (see {Handlers::Processor#globals})

@yieldreturn [Boolean] if the block returns false, parsing is

cancelled.

@return [Proc] the yielded block @see ::after_parse_list @see ::before_parse_file @since 0.7.0

# File lib/yard/parser/source_parser.rb, line 233
def before_parse_list(&block)
  before_parse_list_callbacks << block
end

@return [Array<Proc>] the list of callbacks to be called before

parsing a list of files. Should only be used for testing.

@since 0.7.0

# File lib/yard/parser/source_parser.rb, line 330
def before_parse_list_callbacks
  @before_parse_list_callbacks ||= []
end

@overload initialize(parser_type = ::parser_type, globals = nil)

Creates a new parser object for code parsing with a specific parser type.

@param [Symbol] parser_type the parser type to use
@param [OpenStruct] globals global state to be re-used across separate source files

# File lib/yard/parser/source_parser.rb, line 405
def initialize(parser_type = SourceParser.parser_type, globals1 = nil, globals2 = nil)
  globals = [true, false].include?(globals1) ? globals2 : globals1
  @file = '(stdin)'
  @globals = globals || OpenStruct.new
  self.parser_type = parser_type
end

Parses a path or set of paths

@param [String, Array<String>] paths a path, glob, or list of paths to

parse

@param [Array<String, Regexp>] excluded a list of excluded path matchers @param [Fixnum] level the logger level to use during parsing. See

{YARD::Logger}

@return [void]

# File lib/yard/parser/source_parser.rb, line 99
def parse(paths = DEFAULT_PATH_GLOB, excluded = [], level = log.level)
  log.debug("Parsing #{paths.inspect} with `#{parser_type}` parser")
  excluded = excluded.map do |path|
    case path
    when Regexp; path
    else Regexp.new(path.to_s, Regexp::IGNORECASE)
    end
  end
  files = [paths].flatten.
    map {|p| File.directory?(p) ? "#{p}/**/*.{rb,c,cc,cxx,cpp}" : p }.
    map {|p| p.include?("*") ? Dir[p].sort_by {|d| [d.length, d] } : p }.flatten.
    reject {|p| !File.file?(p) || excluded.any? {|re| p =~ re } }

  log.enter_level(level) do
    parse_in_order(*files.uniq)
  end
end

Parses a string content

@param [String] content the block of code to parse @param [Symbol] ptype the parser type to use. See {parser_type}. @return the parser object that was used to parse content

# File lib/yard/parser/source_parser.rb, line 122
def parse_string(content, ptype = parser_type)
  new(ptype).parse(StringIO.new(content))
end

@return [Hash] a list of registered parser type extensions @private @since 0.5.6

# File lib/yard/parser/source_parser.rb, line 162
def parser_type_extensions; @@parser_type_extensions ||= {} end

Finds a parser type that is registered for the extension. If no type is found, the default Ruby type is returned.

@return [Symbol] the parser type to be used for the extension @since 0.5.6

# File lib/yard/parser/source_parser.rb, line 170
def parser_type_for_extension(extension)
  type = parser_type_extensions.find do |_t, exts|
    [exts].flatten.any? {|ext| ext === extension }
  end
  validated_parser_type(type ? type.first : :ruby)
end

@return [Hash{Symbol=>Object}] a list of registered parser types @private @since 0.5.6

# File lib/yard/parser/source_parser.rb, line 156
def parser_types; @@parser_types ||= {} end

Registers a new parser type.

@example Registering a parser for “java” files

SourceParser.register_parser_type :java, JavaParser, 'java'

@param [Symbol] type a symbolic name for the parser type @param [Base] parser_klass a class that implements parsing and tokenization @param [Array<String>, String, Regexp] extensions a list of extensions or a

regex to match against the file extension

@return [void] @see Parser::Base

# File lib/yard/parser/source_parser.rb, line 145
def register_parser_type(type, parser_klass, extensions = nil)
  unless Base > parser_klass
    raise ArgumentError, "expecting parser_klass to be a subclass of YARD::Parser::Base"
  end
  parser_type_extensions[type.to_sym] = extensions if extensions
  parser_types[type.to_sym] = parser_klass
end

Tokenizes but does not parse the block of code

@param [String] content the block of code to tokenize @param [Symbol] ptype the parser type to use. See {parser_type}. @return [Array] a list of tokens

# File lib/yard/parser/source_parser.rb, line 131
def tokenize(content, ptype = parser_type)
  new(ptype).tokenize(content)
end

Returns the validated parser type. Basically, enforces that :ruby type is never set if the Ripper library is not available

@param [Symbol] type the parser type to set @return [Symbol] the validated parser type @private

# File lib/yard/parser/source_parser.rb, line 183
def validated_parser_type(type)
  !defined?(::Ripper) && type == :ruby ? :ruby18 : type
end

Parses a list of files in a queue.

@param [Array<String>] files a list of files to queue for parsing @return [void]

# File lib/yard/parser/source_parser.rb, line 363
def parse_in_order(*files)
  global_state = OpenStruct.new

  return if before_parse_list_callbacks.any? do |cb|
    cb.call(files, global_state) == false
  end

  OrderedParser.new(global_state, files).parse

  after_parse_list_callbacks.each do |cb|
    cb.call(files, global_state)
  end
end

The main parser method. This should not be called directly. Instead, use the class methods {parse} and {parse_string}.

@param [String, read, Object] content the source file to parse @return [Object, nil] the parser object used to parse the source

# File lib/yard/parser/source_parser.rb, line 417
def parse(content = __FILE__)
  case content
  when String
    @file = File.cleanpath(content)
    content = convert_encoding(String.new(File.read_binary(file)))
    checksum = Registry.checksum_for(content)
    return if Registry.checksums[file] == checksum

    if Registry.checksums.key?(file)
      log.info "File '#{file}' was modified, re-processing..."
    end
    Registry.checksums[@file] = checksum
    self.parser_type = parser_type_for_filename(file)
  else
    content = content.read if content.respond_to? :read
  end

  @contents = content
  @parser = parser_class.new(content, file)

  self.class.before_parse_file_callbacks.each do |cb|
    return @parser if cb.call(self) == false
  end

  @parser.parse
  post_process

  self.class.after_parse_file_callbacks.each do |cb|
    cb.call(self)
  end

  @parser
rescue ArgumentError, NotImplementedError => e
  log.warn("Cannot parse `#{file}': #{e.message}")
  log.backtrace(e, :warn)
rescue ParserSyntaxError => e
  log.warn(e.message.capitalize)
  log.backtrace(e, :warn)
end

Tokenizes but does not parse the block of code using the current {#parser_type}

@param [String] content the block of code to tokenize @return [Array] a list of tokens

# File lib/yard/parser/source_parser.rb, line 461
def tokenize(content)
  @parser = parser_class.new(content, file)
  @parser.tokenize
end

Searches for encoding line and forces encoding @since 0.5.3

# File lib/yard/parser/source_parser.rb, line 470
def convert_encoding(content)
  return content unless content.respond_to?(:force_encoding)
  if content =~ ENCODING_LINE
    content.force_encoding($1)
  else
    content.force_encoding('binary')
    ENCODING_BYTE_ORDER_MARKS.each do |encoding, bom|
      bom.force_encoding('binary')
      if content[0, bom.size] == bom
        content.force_encoding(encoding)
        return content
      end
    end
    content.force_encoding('utf-8') # UTF-8 is default encoding
    content
  end
end

@since 0.5.6

# File lib/yard/parser/source_parser.rb, line 515
def parser_class
  klass = self.class.parser_types[parser_type]
  unless klass
    raise ArgumentError, "invalid parser type '#{parser_type}' or unrecognized file", caller[1..-1]
  end

  klass
end

Guesses the parser type to use depending on the file extension.

@param [String] filename the filename to use to guess the parser type @return [Symbol] a parser type that matches the filename

# File lib/yard/parser/source_parser.rb, line 508
def parser_type_for_filename(filename)
  ext = (File.extname(filename)[1..-1] || "").downcase
  type = self.class.parser_type_for_extension(ext)
  parser_type == :ruby18 && type == :ruby ? :ruby18 : type
end

Runs a {Handlers::Processor} object to post process the parsed statements. @return [void]

# File lib/yard/parser/source_parser.rb, line 490
def post_process
  return unless @parser.respond_to?(:enumerator)

  enumerator = @parser.enumerator
  if enumerator
    post = Handlers::Processor.new(self)
    post.process(enumerator)
  end
end