parse (RobotRules)

    # File lib/spider/robot_rules.rb, line 15
      def parse( text_uri, robots_data )
        uri      = URI.parse(text_uri)
        location = "#{uri.host}:#{uri.port}"
        @rules.delete(location)

        rules      = robots_data.split(/[\015\012]+/).map do |rule|
          rule.sub(/\s*#.*$/, "")
        end
        anon_rules = Array.new
        my_rules   = Array.new
        current    = anon_rules
        rules.each do |rule|
          case rule
          when /^\s*User-Agent\s*:\s*(.+?)\s*$/i
            break unless my_rules.empty?

            current = if $1 == "*"
                        anon_rules
                      elsif $1.downcase.index(@user_agent)
                        my_rules
                      else
                        nil
                      end
          when /^\s*Disallow\s*:\s*(.*?)\s*$/i
            next if current.nil?

            if $1.empty?
              current << nil
            else
              disallow = URI.parse($1)

              next unless disallow.scheme.nil? or disallow.scheme ==
                uri.scheme
              next unless disallow.port.nil?   or disallow.port == uri.port
              next unless disallow.host.nil?   or
              disallow.host.downcase == uri.host.downcase

              disallow = disallow.path
              disallow = "/"            if disallow.empty?
              disallow = "/#{disallow}" unless disallow[0] == ?/

              current << disallow
            end
          end
        end

        @rules[location] = if my_rules.empty?
                             anon_rules.compact
                           else
                             my_rules.compact
                           end
      end