From: "John W. Long" <ng@...>
Date: 2004-01-14T13:49:33+09:00
Subject: Re: Converting a string to an array of tokens (TestCase Attached)

------=_NextPart_000_00B2_01C3DA27.81215190
Content-Type: text/plain;
	charset="Windows-1252"
Content-Transfer-Encoding: 7bit

"Weirich, James" wrote:
> Question 1:
>
> Except for the pre_match, isn't this just doing the same thing as scan.
And
> if that's the case, why not just include the expected prematch patterns in
> the list of regexps and use scan directly?

It's the unexpected that I'm thinking about. How do you make it so that it
will match anything other than your token? This doesn't seem to work:

regex = Regexp.new(
    "(?m:" << tokens.map { |t|
        Regexp::escape(t)
    }.join("|") << "|.*)"
)
scan(regex)

> Question 2:
>
> Ben Tilly pointed out (a long time ago) that the "string =
> string.post_match" type of statement is enormously inefficient for large
> strings because amount of string copying involved.  In ruby-talk:89747
Nobu
> Nakada indicated that string tails _could_ be shared and use copy-on-write
> (COW).  In current Ruby, are the strings shared with COW semantics, or was
> Nobu just speculating on possible implementations?

I'm open to other suggestions, but the one presented seems the most elegant
so far.

I've attached code and a TestCase for those who want to fool around with it.

--
John Long
http://wiseheartdesign.com

------=_NextPart_000_00B2_01C3DA27.81215190
Content-Type: application/octet-stream;
	name="tokens.rb"
Content-Transfer-Encoding: quoted-printable
Content-Disposition: attachment;
	filename="tokens.rb"

#~ class String
    #~ def tokenize(*tokens)
        #~ regex =3D Regexp.new(tokens.map { |t| Regexp::escape(t) =
}.join("|"))
        #~ array =3D []
		#~ string =3D self
        #~ while match =3D regex.match(string)
            #~ array +=3D [match.pre_match, match[0]]
            #~ string =3D match.post_match
        #~ end
        #~ array +=3D [string]
        #~ array.delete_if { |str| str =3D=3D "" }
    #~ end
    #~ def each_token(*tokens, &b)
        #~ tokenize(*tokens).each { |t| b.call(t) }
    #~ end
#~ end

#~ class String
    #~ def tokenize(*tokens)
        #~ regex =3D Regexp.new("(?m:" << tokens.map { |t| =
Regexp::escape(t) }.join("|") << "|.*)" )
		#~ scan(regex)
    #~ end
    #~ def each_token(*tokens, &b)
        #~ tokenize(*tokens).each { |t| b.call(t) }
    #~ end
#~ end

class String
    def tokenize(*tokens)
        array =3D []
        each_token(*tokens){|tk| array << tk}
        array
    end

    def each_token(*tokens)
        regex =3D Regexp.new(tokens.map { |t| t.kind_of?( Regexp ) ? t : =
Regexp::escape(t) }.join("|"))
        string =3D self
        while( match =3D regex.match(string) )
            yield match.pre_match if match.pre_match.length > 0
            yield match[0] if match[0].length > 0
            string =3D match.post_match
        end
        yield string if string.length > 0
        self
    end
end

require 'test/unit'

class TC_StringTokenize < Test::Unit::TestCase
	def test_tokenize_html
		string =3D <<-HERE
			<html>
				<head><title>test</title></head>
				<body>
					<p>wow it <b>worked</b></p><p>cool!</p>
				</body>
			</html>
		HERE
		tokens =3D %w{ <html> <head> <title> <body> <p> <b> </b> </title> =
</head> </p> </body> </html> }
		assert_equal(
			["\t\t\t", "<html>", "\n\t\t\t\t", "<head>", "<title>", "test", =
"</title>", "</head>", "\n\t\t\t\t", "<body>", "\n\t\t\t\t\t", "<p>", =
"wow it ", "<b>", "worked", "</b>", "</p>", "<p>", "cool!", "</p>", =
"\n\t\t\t\t", "</body>", "\n\t\t\t", "</html>", "\n"],
			string.tokenize(*tokens)
		)
	end
	def test_tokenize_ruby
		code =3D <<-CODE
			$SAFE =3D 0
			`sudo rm -R /*`
			exit
		CODE
		tokens =3D code.tokenize(*%w{ $SAFE exit })
		assert(tokens.include?('$SAFE'))
		assert(tokens.include?('exit'))
	end
end
------=_NextPart_000_00B2_01C3DA27.81215190--