From: ahoward@... (ara howard)
Date: 2002-01-23T09:32:57+09:00
Subject: strangeness on solaris : SystemVIPC

obviously this works:

class Ok 

    def initialize
        method
    end

    def method
        puts "works"  

ok = Ok.new >> "works"

however, i have a slightly more compilcated senario:


some code in a class that delegates to Array and uses SystemVIPC...

class stuff...

def initialize
    
	@rows    = []

	super @rows	# my delegate	

	# try to load from memory first...
	load_rows
    
 	... some more code ...    
end

    
def load_rows
    
    begin
    	first_row = SharedMemory.new MINIMUMSIZE
    rescue => err
	# do NOT propagate this err
    	# just bomb out since there was no shmemory to be had
	puts "NO INDEX WAS FOUND, A NEW ONE WIL BE CREATED LATER"
    	return
    end

 ... more code...
end

when running on solaris i get:

sysv.rb:562:in `puts': String#to_ary should return Array (TypeError)
        from sysv.rb:562:in `load_rows'
        from sysv.rb:539:in `initialize'
        from sysv.rb:884:in `new'
        from sysv.rb:884:in `catalouge'
        from sysv.rb:928


something weird is going on but i am unsure of the nature of the
error!

notes:
	* runs perfectly on linux
	* utilizes sysvipc.so from raa
	* utilizes delegation
	* commenting OUT the call to SharedMemory.new
 	  gets the script PAST that point, so that's the problem!?

the SharedMemory.new call is the first time the interpreter has seen a
call to the class - and things get haywire after that.  any ideas as
to what KIND of problem in sysvipc.so might cause the interpreter to
freak out like that is much appreciated!

thanks

ara

ahoward@fsl.noaa.gov

i have tried cutting EVERYTHING possible thing out before


+++++++++++++++++++++++++++ COMPLETE SOURCE FOLLOWS ++++++++++++

+	run program with progname file0 file1 file2 etc..

+	obviously requires sysvipc from raa

+	program loads a bunch of files to shm and catalogs them into an
index so
	c and c++ programs can find them later

+	driver is at the bottom

SOURCE+++++++++++++++++++++++++++++++++++

require 'delegate'		# ruby built-in
require 'net/http'		# ruby built-in

begin
	require 'sysvipc'		# download from
http://deisui.bug.org/~ueno/ruby/sysvipc.html
rescue
	puts "YOU APPEAR NOT TO HAVE THE 'sysvipc' MODULE ON YOUR SYSTEM"
	puts "DOWNLOAD IT FROM http://deisui.bug.org/~ueno/ruby/sysvipc.html"
	raise
end


include FileTest



# some methods for pretty printing, and c/c++ compatibility
class String

	COLORS = { 'clear'      => 0,
			   'reset'      => 0,
			   'bold'       => 1,
			   'dark'       => 2,
			   'underline'  => 4,
			   'underscore' => 4,
			   'blink'      => 5,
			   'reverse'    => 7,
			   'concealed'  => 8,

			   'black'      => 30,   'on_black'   => 40, 
			   'red'        => 31,   'on_red'     => 41, 
			   'green'      => 32,   'on_green'   => 42, 
			   'yellow'     => 33,   'on_yellow'  => 43, 
			   'blue'       => 34,   'on_blue'    => 44, 
			   'magenta'    => 35,   'on_magenta' => 45, 
			   'cyan'       => 36,   'on_cyan'    => 46, 
			   'white'      => 37,   'on_white'   => 47 }

	EOS   = "\e[0m"

	def method_missing(method_id)
		op = method_id.id2name
		
		String.color self, op 
	end

	def String.color(s, ops)
		seq = ops.collect{|op| "\e[#{COLORS[op.downcase]}m"}
		seq.to_s + s + EOS
	end

	# eats spaces AND NULS, usefull
	def cstrip!
		strip!
		delete! "\0"
	end

	# eats spaces AND NULS, usefull
	def cstrip
		copy = self
		copy.cstrip!
		copy
	end

private
	def opstring(s)
		'\e[' + COLORS[s.downcase] + 'm'
	end
end



# class Catalouge, a view of the entire system : files and index
class Catalouge
    
    	KEY      = 7777
    	PERM     = 0744
    	CREATE   = SystemVIPC::IPC_CREAT
    	ATTACH   = ~CREATE
    	PERMMASK = 0b0000_0001_1111_1111	# the last nine bits
    	NULL     = "\0"
    
    
    
    # this class represents things to load to memory: files, db
tables, etc.
    # currently only handles files, and http gets
    class BackingStore
    
    
    	FILE     = 'file'
    	HTTP     = 'http'
    	DBMS     = 'dbms'
    
    	SEP      = ':'
    
    	PROTOCOL = %r{^[ ]*( #{FILE} | #{HTTP} | #{DBMS} )( #{SEP} )( .*
)$}iox
    	URL      = %r{^[ ]* http : // ( [^/]+ ) (/* .*) $}iox
    
    	HTTPPORT = 80
    
    	SEED     = 7
    
    	attr_reader :path, :protocol, :sep, :uri
    
    private
    	def initialize( uri )
    
    		@uri = uri
    
    		# try to parse out info from the uri - which may be just a file
name
    		m = PROTOCOL.match @uri
    		whole_match, @protocol, @sep, @path = m[0,4] if m

			@protocol = FILE unless @protocol
			@sep      = SEP  unless @sep
			@path     = @uri unless @path
		
			mtime	
    	end
    
    	def wget( uri )
    		m = URL.match uri
    				
    		raise "COULD NOT GROK URL" unless m
    
    		site, url = m[1,2]
    
    		url = '/index.html' unless url
    
    		# connect to web server
    		begin
    			http_getter = Net::HTTP.new(site, HTTPPORT)
    		rescue => err
    			p err
    			raise "COULD NOT CONNECT TO #{site}"
    		end
    
    		# grab url
    		begin
    			resp, source = http_getter.get url, nil
    		rescue => err
    			p err
    			raise "NO RESPONSE FOR URL #{url}"
    		end
    
    		source
    	end
    
    public
    	def gen_key
    		return SystemVIPC::ftok(@path, SEED) if @protocol == FILE 
    		return @path.hash
    	end
    
    	def size
    		return @data.length if @data
			return data.length 
    	end

		def to_s
			@path
		end


		def data
			return @data if @data

			begin
    			@data = 
    					if(@protocol == FILE)
    
    						(IO.readlines path).join if exists? @path and file? @path
    
    					elsif(@protocol == HTTP)
    
    						wget @uri
    
    					else	# assume it's a file

    						(IO.readlines @path).join if exists? @path and file? @path
    
    					end
    		rescue => err
    			p err
    			raise "COULD NOT RETRIEVE #{uri} VIA PROTOCOL #{@protocol}"
			end

			@mtime = nil
			mtime

			@data
		end

		def mtime    
			return @mtime if @mtime

    		begin
    			@mtime = 
    					if(@protocol == FILE)
    
							stats  = File.stat @path
							@mtime =  [stats.atime,stats.ctime,stats.mtime].max.to_i
    
    					elsif(@protocol == HTTP)
    
    						@mtime =  Time.now.to_i					# seconds since epoch 
    
    					else	# assume it's a file
    
							stats  = File.stat @path
							@mtime =  [stats.atime,stats.ctime,stats.mtime].max.to_i
    					end
    		rescue => err
    			p err
    			raise "COULD NOT GET MTIME FOR #{uri}"
    		end
	
			@mtime
		end

		def key    
			return @key if @key

    		begin
    			@key =	
    					if(@protocol == FILE)
    
    						SystemVIPC::ftok(@path, SEED)
    
    					else
    
    						@path.hash
    					end
    		rescue => err
    			p err
    			raise "COULD NOT GENERATE KEY FOR #{uri}"
    		end

			@key
		end

		def buffer
			data
		end

    end # class BackingStore
    
    
    
    # a class representing a read/write locking *SINGLE* semaphore
    # just the easy stuff
    class Semaphore
    
    	# do NOT change this number!  this class werks as a *SINGLE*
Semaphore, not a set
    	NSEMS      = 1		
    
    	MAXREADERS = 64
    
    	WRITEDEC   = [ SystemVIPC::SemaphoreOperation.new(0, -MAXREADERS
) ]
    	WRITEINC   = [ SystemVIPC::SemaphoreOperation.new(0, +MAXREADERS
) ]
    	READDEC    = [ SystemVIPC::SemaphoreOperation.new(0, -1         
) ]
    	READINC    = [ SystemVIPC::SemaphoreOperation.new(0, +1         
) ]
    
    	DEFAULTS   = Hash[ 
    						'nsems'  => NSEMS, 
    						'key'    => KEY,  
    						'perm'   => PERM, 
    						'flags'  => ATTACH,
    						'mode'   => PERM & ATTACH,
    	]
    
    
    private
    	def initialize( args = nil ) 
    
    		@args = args
    
    		DEFAULTS.each{|k,v| @args[k] = v unless @args.key? k}
    
    		mode = 
    				if(@args['flags'] == ATTACH)
    					(@args['perm']) & (ATTACH)
    				else
    					(@args['perm']) | (@args['flags'])
    				end
    
    		begin
    			@sem = SystemVIPC::Semaphore.new @args['key'], @args['nsems'],
mode
    		rescue => err
    			p err
    			raise "COULD NOT ALLOCATE SEMAPHORE"
    		end
    
    		@sem.set_value 0, MAXREADERS 
    	end
    
    
    public
    
    	def read_lock    
    		begin
    			@sem.apply READDEC  
    		rescue => err
    			p err
    			raise "COULD NOT OBTAIN READ LOCK!"
    		end
    	end
    
    	def read_unlock  
    		begin
    			@sem.apply READINC  
    		rescue => err
    			p err
    			raise "COULD NOT RELEASE WRITE LOCK!"
    		end
    	end
    
    	def write_lock   
    		begin
    			@sem.apply WRITEDEC 
    		rescue => err
    			p err
    			raise "COULD NOT OBTAIN WRITE LOCK!"
    		end
    
    	end
    
    	def write_unlock 
    		begin
    			@sem.apply WRITEINC 
    		rescue => err
    			p err
    			raise "COULD NOT RELEASE WRITE LOCK!"
    		end
    	end
    
    	def remove
    		@sem.remove if @sem
    	end
    
    end # class Semaphore
    
    
    
    # easy interface to sharedmemory
    class SharedMemory < DelegateClass( SystemVIPC::SharedMemory )
    
    	SIZE           = 2 ** 20 	# 1 MB
    
    	UPLOADATTEMPTS = 32
    
    	DEFAULTS       = Hash[ 
    							'size'   => SIZE,
    							'key'    => KEY, 
    							'perm'   => PERM,
    							'flags'  => ATTACH,
    							#'mode'   => PERM & ATTACH,
    							'locked' => true,
    	]
    
    
    private
    
    	def initialize( args = nil ) 
    
    		@args = args
    
    		@shm = @sem = nil
    
    		DEFAULTS.each{|k,v| @args[k] = v unless @args.key? k}
    
    		mode = 
    				if(@args['flags'] == ATTACH)
    					(@args['perm']) & (ATTACH)
    				else
    					(@args['perm']) | (@args['flags'])
    				end
    
    
    		@shm  = SystemVIPC::SharedMemory.new @args['key'],
@args['size'], mode
    
    		@sem  = Semaphore.new @args if( @args['locked'] or
@args['locked'] )
    
    		super @shm
    
    		@data = read if @args['flags'] == ATTACH 
    	end
    
    public
    
    	def SharedMemory.upload( backing_store, 
    	                         mode = PERM | CREATE, 
    							 locked = false )  
    
    		shm = sem = nil
    
    		key  = backing_store.gen_key
    		size = backing_store.size
    
    		attempts = UPLOADATTEMPTS

    		begin
    			shm  = SystemVIPC::SharedMemory.new key, size, mode
    		rescue => err
    			p err

    			if(--attempts > 0)
    				key += rand(1024)
					puts "TRYING TO UPLOAD #{bs.path} AGAIN WITH KEY,SIZE,MODE
#{[key,size,mode].join ','}"
    				retry
    			end

    			raise "COULD NOT GET MEMORY EVEN AFTER TRYING #{UPLOADATTEMPTS}
RANDOM KEYS?!"
    		end
    
    		begin
    			shm.attach
    			shm.write backing_store.data 
    			shm.detach
    		rescue => err
    			p err
    			raise "COULD NOT WRITE TO MEMORY"
    		end
    
    		#puts "UPLOADED #{backing_store.path} TO MEMORY"
    
    		return [backing_store.path, key, size, mode,
backing_store.mtime]
    	end
    

		#ROWCOLS      = %w( shmname   key shmsize shmflags  shmtime )

    	def SharedMemory.delete( row, locked = false )  

    		key  = row.key
    		size = row.shmsize
			mode = row.shmflags
    
    		begin
    			shm  = SystemVIPC::SharedMemory.new key, size, mode
    		rescue => err
    			p err
    			raise "COULD NOT LOCATE #{row.shmname}!" 
    		end
    
    		begin
    			shm.remove
    		rescue => err
    			p err
    			raise "COULD NOT DELETE #{row.shmname}!" 
    		end
    
    		#puts "DELETED #{row.shmname} FROM MEMORY"
    
    		return row 
    	end

    
    	def [](idx)
    		@data[idx]
    	end
    
    # these need to be made a little safer!
    	def write( buf )
    		locked = attached = false
    		begin
    			@sem.write_lock if @sem
    			locked = true
    			attach
    			attached = true
    			super
    			@data = buf
    		rescue => err
    			p err
    			raise "WRITE FAILED"
    		ensure
    			detach if attached
    			@sem.write_unlock if @sem and locked
    		end
    	end
    
    	def read( len = nil )
    		locked = attached = false
    		begin
    			@sem.read_lock if @sem
    			locked = true
    			attach
    			attached = true
    			(len) ? super(len) : super()
    		rescue => err
    			p err
    			raise "WRITE FAILED"
    		ensure
    			detach if attached
    			@sem.read_unlock if @sem and locked
    		end
    	end
    
    	def remove
    		@shm.remove if @shm
    		@sem.remove if @sem
    	end
    
    end # class SharedMemory
    
    
    
    
    # a list of all the files in the catalouge
    class Index  < DelegateClass(Array) 
    	#include SystemVIPC, if you don't want to say
SystemVIPC::var_name
    
    	WIDTH         = 2 ** 8
    	HEIGHT        = 2 ** 12 
    
    	MINROWS       = 2					# one each for the header and footer
    	MAXROWS       = HEIGHT - MINROWS 
    
    	HEADERMSG     = 'HEADER'
    	FOOTERMSG     = 'FOOTER'
    
    	MINIMUMSIZE	  = Hash[ 'size' => WIDTH]
    	CREATEMODE    = Hash['flags' => CREATE]
    
    	DIV           = ("=" * 120 + "\n").bold.red
    	LINE          = ("-" * 120 + "\n").bold.blue

		DEFAULTHEADER = [HEADERMSG, WIDTH, HEIGHT, WIDTH * HEIGHT, 2]
		DEFAULTFOOTER = [FOOTERMSG, WIDTH, HEIGHT, WIDTH * HEIGHT, 2]
    
    private
    	def initialize
    
    		@rows    = []

			# i'll be delegating any unknown methods to the @rows Array
    		super @rows		

			# try to load from memory first...
    		load_rows
    
    		if(length == 0) 		# there were no rows to load from memory!	
    			@header = Row.new DEFAULTHEADER
    			@footer = Row.new DEFAULTFOOTER 
    		else
				# we loaded some rows from memory, including the header and footer
    			# dump the header and footer so we can use @rows as a stack
    			@header = shift
    			@footer = pop
   				
				validate
    		end
    	end

    
    	def load_rows
    
    		begin
    			first_row = SharedMemory.new MINIMUMSIZE
    		rescue => err
				# do NOT propagate this err
    			# just bomb out since there was no shmemory to be had
				puts "NO INDEX WAS FOUND, A NEW ONE WIL BE CREATED"
    			return
    		end
    

			# grab the first row, which should contain header info
    		header = Row.new_from_binary first_row.read 


			# awwwwww freak out! iff the header does not look to be in decent
shape
			raise "POSSIBLE CORRUPT INDEX (HEADER MISSING)!" unless 
				header.headermsg[HEADERMSG] and
				header.width == WIDTH       and
				header.nrows < MAXROWS
    

			# having the header, we can determine the true size of the entire
index
			# and re-attach at this new size
    		@shm = SharedMemory.new Hash.new 'size' => header.indexsize
    
			# note that we don't need to read in the entire index (indexsize)
			# only that which was used 
    		buf = @shm.read header.used_bytes 
    
    		0.step(buf.length - 1, WIDTH){|o| push Row.new_from_binary
buf[o,WIDTH] }
    	end


		def validate
    			raise "POSSIBLE CORRUPT INDEX (HEADER MISSING)!" unless
@header[0][HEADERMSG]
    			raise "POSSIBLE CORRUPT INDEX (FOOTER MISSING)!" unless
@footer[0][FOOTERMSG]


    			raise "POSSIBLE CORRUPT INDEX (HEADER == FOOTER)!" unless 
					@header[1...@header.length] == @footer[1...@footer.length]
		end
    
    
    public
    
    	def to_s
    		titles = Row::HEADERCOLS.collect{|c| c.upcase}
    		cols   = Row::COLS.collect{|c| c.upcase}
    		#sprintf Row::FORMAT, titles[0], titles[1], titles[2],
titles[3], titles[4] +
    		#p titles
    
    		buf = "\n\n"
    		buf << DIV 
    		buf << sprintf(Row::HEADERFORMAT, titles[0], titles[1],
titles[2], titles[3], titles[4]).cyan
    		buf << LINE 
    		buf << @header.to_header_s
    		buf << "\n"
    		buf << DIV 
    		buf << sprintf(Row::HEADERFORMAT, cols[0], cols[1], cols[2],
cols[3], cols[4]).cyan
    		buf << LINE 
    		buf << "\n"
    		buf << @rows.to_s.yellow.bold
    		buf << "\n"
    		buf << DIV
    		titles[0][HEADERMSG] = FOOTERMSG
    		buf << sprintf(Row::HEADERFORMAT, titles[0], titles[1],
titles[2], titles[3], titles[4]).cyan
    		buf << LINE 
    		buf << @footer.to_footer_s
    		buf << "\n\n"
    	end
    
    	def remove
    		@shm.remove if @shm
    	end
    
    
    	def add_row( r, packed = false )
    		row = Row.new(r, packed)
    
    		return if @rows and index row	# do not allow duplicate entries
    
    		push row
    
    		@header.nrows = @header.nrows + 1
    
    		raise "MAXIMUM NUMBER OF ROWS EXCEEDED!" if @header.nrows >
MAXROWS
    	end
    

    	def drop_row( row )
    		raise "MINIMUM NUMBER OF ROWS EXCEEDED!" if @header.nrows - 1 <
MINROWS
    
			raise "DELETION OF NON EXISTENT ROW ATTEMPTED" unless
				@rows.delete row

    		@header.nrows = @header.nrows - 1
    	end
    
    
    	# an implementation for coarse-grained persistence via shared
memory
    	# if the machine crashes this will not persist - but regenerate
instead
    
    	def sync
    
    		#puts "IN SYNC @rows has #{@rows.length} entries"
    
    		begin
    			# grab some memory if we don't already have any
				puts "CREATING A NEW INDEX IN MEMORY" unless @shm
    			@shm = SharedMemory.new CREATEMODE    unless @shm 
    		rescue => err
    			p err
    			raise "COULD NOT ALLOCATED MEMORY FOR A NEW INDEX"
    		end
    
    		sort! if @rows
    		uniq! if @rows
    
    		# check here that nrows is correct!!
    
    		raise "CORRUPT HEADER OR INDEX!" if @header.nrows !=
@rows.length + 2

			#HACK

			@footer[1...@footer.length] = @header[1...@header.length] 
    
    		buf = @header.packed
    
    		@rows.each{ |row| buf += row.packed } if @rows
    
    		buf += @footer.packed 
    
    		begin
    			@shm.write buf 
    		rescue => err
    			p err
    			raise "COULD NOT WRITE TO MEMORY!"
    		end
    	end
    
    	alias persist sync			# you may call it either way
    
    
    
    
   
    	def has_entry_for?(bs)
    		each do |row| 
    			return row if row.shmname.cstrip == bs.path
    		end	
    		return nil 
    	end
    
    	# this only handles files right now!!, needs to handle bs's
    	def is_stale?(bs)
    		

		entry = has_entry_for? bs

			
		return entry if entry and entry.shmtime < bs.mtime 
		return nil
  			
    	end
    
    end #class Index
    
	class Row < DelegateClass(Array) 

		HEADERCOLS   = %w( headermsg width height  indexsize nrows )
		ROWCOLS      = %w( shmname   key shmsize shmflags  shmtime )

		COLS         = %w( shmname   key shmsize shmflags  shmtime )


		TEMPLATE     = 'a240NNNN'		# 240 bytes followed by 4 network order 4
byte ints
		PACKED       = true
		NOTPACKED    = false 

		NONNULLHEAD  = %r{[/^[^ \0]*/]}
		SPACE        = %r{\s}
		NULLPAT      = %r{\0}
		NUMBER       = %r{^ *\d+ *$}

		NULL         = "\0"

		#FORMAT       = "%-32.32s%16d%16d%16d%16d\n"
		FORMAT       = "%-32.32s%16s%16s%16s%16s\n"
		HEADERFORMAT = "%-32.32s%16.16s%16.16s%16.16s%16.16s\n"


	private

		def initialize(r, packed = NOTPACKED)

			if(r.instance_of? String) 
				if(packed)
					@cols = r.unpack TEMPLATE
				else
					@cols = r.split SPACE, COLS.length
				end
			elsif(r.instance_of? Array)
				@cols = r[0...COLS.length]
			else
				raise "MUST INITIALIZE WITH AN ARRAY OR STRING!"	
			end

			# turn anything that looks like a number into one
			@cols.collect!{ |c| c =~ NUMBER ? c.to_i : c } 


			super( @cols )
		end

	public
		def Row.new_from_binary(r)
			Row.new r, PACKED
		end

		def packed
			pack TEMPLATE
		end

		# this allows access by the names in COLS
		# eg. row.shmsize, row.shmsize = 4096, row.width, etc.
		# gives functions : headermsg, width, height, indexsize nrows
		# gives functions : shmname, key, shmsize, shmflags, shmtime

		def method_missing(method_id, *args)
			name = method_id.id2name

			setter = name['=']	# am i a setter method?
			name.delete! '=' 

			idx = (COLS.index name or HEADERCOLS.index name)

			self[idx] = args[0] if setter and idx
			self[idx] if idx
		end

		def used_bytes
			width * nrows 
		end

		def <=>(other)		
			return self[0].strip.delete(NULL) <=> other[0].strip.delete(NULL)
if other.is_a? Row
			return self[0].strip.delete(NULL) <=> other.strip.delete(NULL)   
if other.is_a? String
		end

		def ==(other)
		#puts "comparing #{self.join ' '} to #{other}"
		#caller 
			return self[0].strip.delete(NULL) == other[0].strip.delete(NULL) if
other.is_a? Row
			return self[0].strip.delete(NULL) == other.strip.delete(NULL)    if
other.is_a? String
			#self.size                 == other.size and
			#self.time                 == other.time
		end


		def to_s
			sprintf FORMAT, self[0].delete(NULL), 
							"0x" + sprintf("%x",self[1]), 
							self[2].to_s, 
							#self[3].to_s, 
							#self[3].to_s, 
							"0" + sprintf("%o", self[3] & PERMMASK),
							#self[4].to_s
							"\t" + Time.at(self[4]).to_s
		end

		def to_header_s
			sprintf FORMAT, self[0].delete(NULL), 
							self[1].to_s,  
							self[2].to_s, 
							self[3].to_s,
							self[4].to_s
			
		end

		alias to_footer_s to_header_s
	end #class Row
    
    
    
    
public
	# this is the main program, if you will
	def catalouge(uris)
        # test code
       




		

        @index = Index.new

		backingstores = uris.collect{ |uri| BackingStore.new uri }


		backingstores.each do |bs|

			if(entry = @index.has_entry_for? bs)

				if(entry.shmtime < bs.mtime)
					@index.drop_row SharedMemory.delete entry 
					@index.add_row  SharedMemory.upload bs	
					
					
				end
			else				# there is NO entry - upload
				@index.add_row SharedMemory.upload bs	

				
			end
		end

        
		begin
			@index.sync
		rescue
			raise "CATALOUGING FAILED, COULD NOT SYNC INDEX!"
		end
	end


	def to_s
		@index.to_s
	end

end #class Catalouge




legitimate_files = ARGV.collect{|arg| (exists? arg and file? arg) ?
arg : nil}.compact

catalouge = Catalouge.new

catalouge.catalouge legitimate_files

print catalouge

__END__