module UTFtests

  def utf8?(reader, lvl)
  # reader - muss 'each_with_address'-Methode haben, die zwei Werte an den Block
  # uebergibt. Einen Adresswert und ein Byte als Zahlenwert.
  # Level gibt den Report-Level an. Falls ein Block vorhanden ist wird er mit zwei Parametern
  # versorgt. Den aktuellen Level und ein dazu passendes Array. Der Block wird nur aufgerufen, wenn
  # 'lvl' kleiner oder gleich dem yield-level ist.
  # level 0 - [addr, byte] - fuer jedes verarbeitete Byte
  # level 1 - [Codepoint] - fuer jeden korrekten Unicode-Codepoint
  # level 2 - [n_of_bytes, n_of_chars] - Anzahl verarbeiteter Bytes und Anzahl korrekter Unicode-Codepoints
  # level 3 - [text] Textmeldung: BOM gefunden, Fehler sind aufgetreten, ...
  # level 4 - [addr, byte] - Byte 'byte' an falscher Position bei Adresse 'addr'
    first = true
    errors = false
    bytes = Array.new(3)
    charerror = false
    foundBOM = false
    n_of_chars = 0
    n_of_bytes = 0
    block = block_given??true:false
    state = 'Start'

    reader.each_with_address do |addr, byte|
    
      n_of_bytes += 1
      yield 0, [addr, byte] if block && lvl <= 0
    
      # Suche Wiederaufsatzpunkt nach fehlerhaftem Byte
      if charerror && (byte <= 0x7F || byte >= 0xC2) && byte <= 0xF4
        charerror = false
        errors = true
        state = 'Start'
      elsif charerror
        errors = true
        next
      end
      
      # Der Automat als Case-Construct
      case state
    
        # Initialer State und neues Zeichen erwartet
        when 'Start'
          bytes[0] = byte
          case byte
            when 0xEF
              if first
                state = 'BOMmiddle'
              else
                state = 'ThreeByteMiddle2'
              end
            when 0x00..0x7F
              n_of_chars += 1
              yield 1, [bytes[0]] if block && lvl <= 1
            when 0xC2..0xDF
              state = 'TwoByteLast'
            when 0xE0
              state = 'ThreeByteMiddle1'
            when 0xE1..0xEF
              state = 'ThreeByteMiddle2'
            when 0xF0
              state = 'FourByteSecond1'
            when 0xF1..0xF4
              state = 'FourByteSecond2'
            else
              charerror = true
          end # case byte

        # Eventuell kommt mittleres BOM-Byte
        when 'BOMmiddle'
          bytes[1] = byte
          case byte
            when 0xBB
              state = 'BOMlast'
            when 0x80..0xBF
              state = 'ThreeByteLast'
            else
              charerror = true
          end # case byte

        # Eventuell BOM gefunden
        when 'BOMlast'
          case byte
            when 0xBF
              state = 'Start'
              foundBOM = true
              yield 3, ["Found a UTF-8 Byte Order Mark (BOM)"] if block && lvl <= 3
            when 0x80..0xBF
              state = 'Start'
              n_of_chars += 1
              yield 1, [((bytes[0] & 0x0F) << 12) + ((bytes[1] & 0x3F) << 6) +
                                     (byte & 0x3F)] if block && lvl <= 1
            else
              charerror = true
          end # case byte

        # Zweites und letztes Byte erwartet
        when 'TwoByteLast'
          case byte
            when  0x80..0xBF
              state = 'Start'
              n_of_chars += 1
              yield 1, [((bytes[0] & 0x1F) << 6) + (byte & 0x3F)] if block && lvl <= 1
            else
              charerror = true
          end # case byte

        # Mittleres Byte erwartet
        when 'ThreeByteMiddle1' 
          bytes[1] = byte
          case byte
            when 0xA0..0xBF
              state = 'ThreeByteLast'
            else
              charerror = true
          end # case byte

        # Mittleres Byte erwartet
        when 'ThreeByteMiddle2' 
          bytes[1] = byte
          case byte
            when 0x80..0xBF
              state = 'ThreeByteLast'
            else
              charerror = true
            end # case byte

        # Drittes und letztes Byte erwartet
        when 'ThreeByteLast'
          case byte
            when 0x80..0xBF
              state = 'Start'
              n_of_chars += 1
              yield 1, [((bytes[0] & 0x0F) << 12) +((bytes[1] & 0x3F) << 6) +
                                     (byte & 0x3F)] if block && lvl <= 1
            else
              charerror = true
          end # case byte

        # Zweites von vier Bytes erwartet
        when 'FourByteSecond1'
          bytes[1] = byte
          case byte
            when 0x90..0xBF
              state = 'FourByteThird'
            else
              charerror = true
          end # case byte

        # Zweites von vier Bytes erwartet
        when 'FourByteSecond2'
          bytes[1] = byte
          case byte
            when 0x80..0xBF
              state = 'FourByteThird'
            else
              charerror = true
          end # case byte

        # Drittes von vier Bytes erwartet
        when 'FourByteThird'
          bytes[2] = byte
          case byte
            when 0x80..0xBF
              state = 'FourByteLast'
            else
              charerror = true
          end # case byte

        # Viertes und letztes von vier Bytes erwartet
        when 'FourByteLast'
          case byte
            when 0x80..0xBF
              state = 'Start'
              n_of_chars += 1
              yield 1, [((bytes[0] & 0x07) << 18) + ((bytes[1] & 0x3F) << 12) +
                                    ((bytes[2] & 0x3F) << 6) + (byte & 0x3F)] if block && lvl <= 1
            else
              charerror = true
          end # case byte

        # Unm��glicher Fehler ;-))
        else
          raise "+++ utf8? - internal error: unknown state"
          return false
        
      end # case state
    
      # BOM wird nur am Anfang als solches erkannt
      first = false
      if charerror
        yield 4, [addr, byte] if block && lvl <= 4
      end # if
    
    end # each_with_address
  
    # Start-State ist auch einzig korrekter End-State
    yield 2, [n_of_bytes, n_of_chars] if block && lvl <= 2
    if state == 'Start' && !charerror && !errors
      true
    else
    yield 3, ["Incorrect UTF-8 encoded data"] if block && lvl <= 3
      false
    end # if
  
  end # utf8?

end # UTFtests

require "UTFtests.rb"
include UTFtests

class Ostr
  def initialize(str)
    @buff = str
  end
  def each_with_address
    position = -1
    @buff.each_byte{|byte|yield position+=1, byte}
  end
end

erg = utf8?(Ostr.new("\xEF\xBB\xBFblabla\xE0\x9F\x80blabla\xEF\xBB\xBF"), 0) do |lvl, inf|
  case lvl
    when 0
      puts "Adress: #{"%08X" % inf[0]}, Byte: #{"%02X" % inf[1]}"
    when 1
      puts "Codepoint #{"%08X" % inf[0]}"
    when 2
      puts "#{inf[0]} Bytes processed, #{inf[1]} correct Codepoints (without a possible BOM)"
    when 3
      puts "#{inf[0]}"
    when 4
      puts "+++ Illegal byte, address: #{"%08X" % inf[0]}, content: #{"%02X" % inf[1]}"
  end
end

puts "utf8?-Ergebnis: #{erg}"

Adress: 00000000, Byte: EF
Adress: 00000001, Byte: BB
Adress: 00000002, Byte: BF
Found a UTF-8 Byte Order Mark (BOM)
Adress: 00000003, Byte: 62
Codepoint 00000062
Adress: 00000004, Byte: 6C
Codepoint 0000006C
Adress: 00000005, Byte: 61
Codepoint 00000061
Adress: 00000006, Byte: 62
Codepoint 00000062
Adress: 00000007, Byte: 6C
Codepoint 0000006C
Adress: 00000008, Byte: 61
Codepoint 00000061
Adress: 00000009, Byte: E0
Adress: 0000000A, Byte: 9F
+++ Illegal byte, address: 0000000A, content: 9F
Adress: 0000000B, Byte: 80
Adress: 0000000C, Byte: 62
Codepoint 00000062
Adress: 0000000D, Byte: 6C
Codepoint 0000006C
Adress: 0000000E, Byte: 61
Codepoint 00000061
Adress: 0000000F, Byte: 62
Codepoint 00000062
Adress: 00000010, Byte: 6C
Codepoint 0000006C
Adress: 00000011, Byte: 61
Codepoint 00000061
Adress: 00000012, Byte: EF
Adress: 00000013, Byte: BB
Adress: 00000014, Byte: BF
Codepoint 0000FEFF
21 Bytes processed, 13 correct Codepoints (without a possible BOM)
Incorrect UTF-8 encoded data
utf8?-Ergebnis: false

require "UTFtests.rb"
include UTFtests

class Fstr
  def initialize(name)
    @buff = File.new(name, "rb").read
  end
  def each_with_address
    position = -1
    @buff.each_byte{|byte|yield position+=1, byte}
  end
end

erg = utf8?(Fstr.new("utf8datei.txt"), 2) do |lvl, inf|
  case lvl
    when 0
      puts "Adress: #{"%08X" % inf[0]}, Byte: #{"%02X" % inf[1]}"
    when 1
      puts "Codepoint #{"%08X" % inf[0]}"
    when 2
      puts "#{inf[0]} Bytes processed, #{inf[1]} correct Codepoints (without a possible BOM)"
    when 3
      puts "#{inf[0]}"
    when 4
      puts "+++ Illegal byte, address: #{"%08X" % inf[0]}, content: #{"%02X" % inf[1]}"
  end
end

puts "utf8?-Ergebnis: #{erg}"

Found a UTF-8 Byte Order Mark (BOM)
51 Bytes processed, 47 correct Codepoints (without a possible BOM)
utf8?-Ergebnis: true

Hall��chen,
dies ist eine UTF-8-codierte Datei 

Wolfgang>utf8_blocks
Correct usage is 'utf8_blocks <filename>'
or               'ruby utf8_blocks.rb <filename>'

Wolfgang>utf8_blocks otto
+++++ File 'otto' does not exist

Wolfgang>utf8_blocks utf8laenger.txt
----------------------------------------------------------------
Start der Analyse der utf-8-codierten Datei 'utf8laenger.txt'
----------------------------------------------------------------
Found a UTF-8 Byte Order Mark (BOM)
----------------------------------------------------------------
Ergebnis der Analyse der utf-8-codierten Datei 'utf8laenger.txt'
----------------------------------------------------------------
BOM or ZWNBSP.....................................       1
Basic Latin.......................................   10807
----------------------------------------------------------------

Wolfgang>utf8_blocks keinutf8.txt
-------------------------------------------------------------
Start der Analyse der utf-8-codierten Datei 'keinutf8.txt'
-------------------------------------------------------------
+++ Illegal byte, address: 00000001, content: F6
-------------------------------------------------------------

Wolfgang>to_unix utf8_blocks.rb

Wolfgang>utf8_blocks-unix keinutf8.txt
-------------------------------------------------------------
Start der Analyse der utf-8-codierten Datei 'keinutf8.txt'
-------------------------------------------------------------
+++ Illegal byte, address: 00000001, content: F6
-------------------------------------------------------------

Wolfgang>utf8_blocks-unix utf8laenger.txt
----------------------------------------------------------------
Start der Analyse der utf-8-codierten Datei 'utf8laenger.txt'
----------------------------------------------------------------
Found a UTF-8 Byte Order Mark (BOM)
----------------------------------------------------------------
Ergebnis der Analyse der utf-8-codierten Datei 'utf8laenger.txt'
----------------------------------------------------------------
BOM or ZWNBSP.....................................       1
Basic Latin.......................................   10807
----------------------------------------------------------------

Wolfgang>

>>> #
>>> # Find used code blocks in input data
>>> #
>>> ./utf8_blocks-unix.rb mappingbased_objects_en.ttl
----------------------------------------------------------------------------
Start der Analyse der utf-8-codierten Datei 'mappingbased_objects_en.ttl'
----------------------------------------------------------------------------
----------------------------------------------------------------------------
Ergebnis der Analyse der utf-8-codierten Datei 'mappingbased_objects_en.ttl'
----------------------------------------------------------------------------
Arabic............................................     831
Arabic Presentation Forms-B.......................       2
Arrows............................................      78
Basic Latin.......................................2449631147
Bengali...........................................      38
CJK Symbols and Punctuation.......................      14
CJK Unified Ideographs............................    2339
Combining Diacritical Marks.......................      48
Currency Symbols..................................      19
Cyrillic..........................................    3259
Devanagari........................................     129
General Punctuation...............................  358693
Greek Extended....................................       3
Greek and Coptic..................................     839
Gujarati..........................................      76
Halfwidth and Fullwidth Forms.....................      85
Hangul Syllables..................................     203
Hebrew............................................      48
Hiragana..........................................     124
IPA Extensions....................................    8773
Katakana..........................................     383
Latin Extended Additional.........................   20903
Latin Extended-A.................................. 1224890
Latin Extended-B..................................   97531
Latin-1 Supplement................................ 2317318
Letterlike Symbols................................      63
Malayalam.........................................     119
Mathematical Operators............................    1259
Miscellaneous Mathematical Symbols-A..............      42
Miscellaneous Symbols.............................     171
Myanmar...........................................       9
Number Forms......................................      67
Sinhala...........................................      39
Small Form Variants...............................       2
Spacing Modifier Letters..........................    1909
Supplemental Mathematical Operators...............       1
Tamil.............................................      48
Telugu............................................      22
Thai..............................................      64
----------------------------------------------------------------------------
>>>