Update midi_processing to latest version, fixing some severe MIDI file handling issues, including Standard MIDI file SysEx and SysEx continuation handling

Frameworks/midi_processing/midi_processing/midi_container.cpp

@@ -95,6 +95,11 @@ const midi_event & midi_track::operator [] ( std::size_t p_index ) const
     return m_events[ p_index ];
 }
 
+midi_event & midi_track::operator [] ( std::size_t p_index )
+{
+    return m_events[ p_index ];
+}
+
 void midi_track::remove_event( unsigned long index )
 {
     m_events.erase( m_events.begin() + index );
@@ -161,6 +166,11 @@ const tempo_entry & tempo_map::operator [] ( std::size_t p_index ) const
     return m_entries[ p_index ];
 }
 
+tempo_entry & tempo_map::operator [] ( std::size_t p_index )
+{
+    return m_entries[ p_index ];
+}
+
 system_exclusive_entry::system_exclusive_entry(const system_exclusive_entry & p_in)
 {
     m_port = p_in.m_port;
@@ -339,8 +349,6 @@ void midi_container::initialize( unsigned p_form, unsigned p_dtx )
         m_timestamp_loop_start.resize( 1 );
         m_timestamp_loop_end.resize( 1 );
     }
-    int port = 0;
-	limit_port_number(port);
 }
 
 void midi_container::add_track( const midi_track & p_track )
@@ -526,7 +534,12 @@ void midi_container::apply_hackfix( unsigned hack )
     }
 }
 
-void midi_container::serialize_as_stream( unsigned long subsong, std::vector<midi_stream_event> & p_stream, system_exclusive_table & p_system_exclusive, unsigned long & loop_start, unsigned long & loop_end, unsigned clean_flags ) const
+void midi_container::serialize_as_stream( unsigned long subsong,
+                                          std::vector<midi_stream_event> & p_stream,
+                                          system_exclusive_table & p_system_exclusive,
+                                          unsigned long & loop_start,
+                                          unsigned long & loop_end,
+                                          unsigned clean_flags ) const
 {
     std::vector<uint8_t> data;
     std::vector<std::size_t> track_positions;
@@ -1056,7 +1069,186 @@ void midi_container::get_meta_data( unsigned long subsong, midi_meta_data & p_ou
     p_out.append( m_extra_meta_data );
 }
 
-void midi_container::scan_for_loops( bool p_xmi_loops, bool p_marker_loops, bool p_rpgmaker_loops )
+void midi_container::trim_tempo_map( unsigned long p_index, unsigned long base_timestamp )
+{
+    if ( p_index < m_tempo_map.size() )
+    {
+        tempo_map & map = m_tempo_map[ p_index ];
+
+        for ( unsigned long i = 0, j = map.get_count(); i < j; ++i )
+        {
+            tempo_entry & entry = map[ i ];
+            if ( entry.m_timestamp >= base_timestamp )
+                entry.m_timestamp -= base_timestamp;
+            else
+                entry.m_timestamp = 0;
+        }
+    }
+}
+
+void midi_container::trim_range_of_tracks(unsigned long start, unsigned long end)
+{
+    unsigned long timestamp_first_note = ~0UL;
+
+    for (unsigned long i = start; i <= end; ++i)
+    {
+        unsigned long j, k;
+
+        const midi_track & track = m_tracks[ i ];
+
+        for (j = 0, k = track.get_count(); j < k; ++j)
+        {
+            const midi_event & event = track[ j ];
+
+            if ( event.m_type == midi_event::note_on && event.m_data[ 0 ] )
+                break;
+        }
+
+        if ( j < k )
+        {
+            if ( track[ j ].m_timestamp < timestamp_first_note )
+                timestamp_first_note = track[ j ].m_timestamp;
+        }
+    }
+
+    if ( timestamp_first_note < ~0UL && timestamp_first_note > 0 )
+    {
+        for (unsigned long i = start; i <= end; ++i)
+        {
+            midi_track & track = m_tracks[ i ];
+
+            for (unsigned long j = 0, k = track.get_count(); j < k; ++j)
+            {
+                midi_event & event = track[ j ];
+                if ( event.m_timestamp >= timestamp_first_note )
+                    event.m_timestamp -= timestamp_first_note;
+                else
+                    event.m_timestamp = 0;
+            }
+        }
+
+        if ( start == end )
+        {
+            trim_tempo_map( start, timestamp_first_note );
+
+            m_timestamp_end[ start ] -= timestamp_first_note;
+
+            if ( m_timestamp_loop_end[ start ] != ~0UL )
+                m_timestamp_loop_end[ start ] -= timestamp_first_note;
+            if ( m_timestamp_loop_start[ start ] != ~0UL )
+            {
+                if ( m_timestamp_loop_start[ start ] > timestamp_first_note )
+                    m_timestamp_loop_start[ start ] -= timestamp_first_note;
+                else
+                    m_timestamp_loop_start[ start ] = 0;
+            }
+        }
+        else
+        {
+            trim_tempo_map( 0, timestamp_first_note );
+
+            m_timestamp_end[ 0 ] -= timestamp_first_note;
+
+            if ( m_timestamp_loop_end[ 0 ] != ~0UL )
+                m_timestamp_loop_end[ 0 ] -= timestamp_first_note;
+            if ( m_timestamp_loop_start[ 0 ] != ~0UL )
+            {
+                if ( m_timestamp_loop_start[ 0 ] > timestamp_first_note )
+                    m_timestamp_loop_start[ 0 ] -= timestamp_first_note;
+                else
+                    m_timestamp_loop_start[ 0 ] = 0;
+            }
+        }
+    }
+}
+
+void midi_container::trim_start()
+{
+    if (m_form == 2)
+    {
+        for (unsigned long i = 0, j = m_tracks.size(); i < j; ++i)
+        {
+            trim_range_of_tracks(i, i);
+        }
+    }
+    else
+    {
+        trim_range_of_tracks(0, m_tracks.size() - 1);
+    }
+}
+
+void midi_container::split_by_instrument_changes(split_callback cb)
+{
+    if (m_form != 1) /* This would literally die on anything else */
+        return;
+
+    for (unsigned long i = 0, j = m_tracks.size(); i < j; ++i)
+    {
+        midi_track source_track = m_tracks[0];
+
+        m_tracks.erase(m_tracks.begin());
+
+        midi_track output_track;
+        midi_track program_change;
+
+        for (unsigned long k = 0, l = source_track.get_count(); k < l; ++k)
+        {
+            const midi_event & event = source_track[ k ];
+            if ( event.m_type == midi_event::program_change ||
+               ( event.m_type == midi_event::control_change &&
+                 (event.m_data[0] == 0 || event.m_data[0] == 0x20)))
+            {
+                program_change.add_event( event );
+            }
+            else
+            {
+                if (program_change.get_count())
+                {
+                    if (output_track.get_count())
+                        m_tracks.push_back( output_track );
+                    output_track = program_change;
+					if (cb)
+					{
+						unsigned long timestamp = 0;
+						uint8_t bank_msb = 0, bank_lsb = 0, instrument = 0;
+						for (int i = 0, j = program_change.get_count(); i < j; ++i)
+						{
+							const midi_event & ev = program_change[i];
+							if (ev.m_type == midi_event::program_change)
+								instrument = ev.m_data[0];
+							else if (ev.m_data[0] == 0)
+								bank_msb = ev.m_data[1];
+							else
+								bank_lsb = ev.m_data[1];
+							if (ev.m_timestamp > timestamp)
+								timestamp = ev.m_timestamp;
+						}
+
+						std::string name = cb(bank_msb, bank_lsb, instrument);
+
+						std::vector<uint8_t> data;
+
+						data.resize(name.length() + 2);
+
+						data[0] = 0xFF;
+						data[1] = 0x03;
+
+						std::copy(name.begin(), name.end(), data.begin() + 2);
+
+						output_track.add_event(midi_event(timestamp, midi_event::extended, 0, &data[0], data.size()));
+					}
+					program_change = midi_track();
+                }
+                output_track.add_event( event );
+            }
+        }
+
+        if (output_track.get_count())
+            m_tracks.push_back(output_track);
+    }
+}
+
+void midi_container::scan_for_loops( bool p_xmi_loops, bool p_marker_loops, bool p_rpgmaker_loops, bool p_touhou_loops )
 {
     std::vector<uint8_t> data;
 
@@ -1071,6 +1263,51 @@ void midi_container::scan_for_loops( bool p_xmi_loops, bool p_marker_loops, bool
         m_timestamp_loop_end[ i ] = ~0UL;
     }
 
+    if ( p_touhou_loops && m_form == 0 )
+    {
+        bool loop_start_found = false;
+        bool loop_end_found = false;
+        bool errored = false;
+
+        for ( unsigned long i = 0; !errored && i < m_tracks.size(); ++i )
+        {
+            const midi_track & track = m_tracks[ i ];
+            for ( unsigned long j = 0; !errored && j < track.get_count(); ++j )
+            {
+                const midi_event & event = track[ j ];
+                if ( event.m_type == midi_event::control_change )
+                {
+                    if ( event.m_data[ 0 ] == 2 )
+                    {
+                        if ( event.m_data[ 1 ] != 0 )
+                        {
+                            errored = true;
+                            break;
+                        }
+                        m_timestamp_loop_start[ 0 ] = event.m_timestamp;
+                        loop_start_found = true;
+                    }
+                    if ( event.m_data[ 0 ] == 4 )
+                    {
+                        if ( event.m_data[ 1 ] != 0 )
+                        {
+                            errored = true;
+                            break;
+                        }
+                        m_timestamp_loop_end[ 0 ] = event.m_timestamp;
+                        loop_end_found = true;
+                    }
+                }
+            }
+        }
+
+        if ( errored )
+        {
+            m_timestamp_loop_start[ 0 ] = ~0UL;
+            m_timestamp_loop_end[ 0 ] = ~0UL;
+        }
+    }
+
     if ( p_rpgmaker_loops )
     {
         bool emidi_commands_found = false;

Frameworks/midi_processing/midi_processing/midi_container.h

@@ -57,6 +57,7 @@ public:
     void add_event( const midi_event & p_event );
     std::size_t get_count() const;
     const midi_event & operator [] ( std::size_t p_index ) const;
+    midi_event & operator [] ( std::size_t p_index );
 
     void remove_event( unsigned long index );
 };
@@ -80,6 +81,7 @@ public:
 
     std::size_t get_count() const;
     const tempo_entry & operator [] ( std::size_t p_index ) const;
+    tempo_entry & operator [] ( std::size_t p_index );
 };
 
 struct system_exclusive_entry
@@ -233,6 +235,17 @@ public:
 
     void promote_to_type1();
 
+    void trim_start();
+
+private:
+    void trim_range_of_tracks(unsigned long start, unsigned long end);
+    void trim_tempo_map(unsigned long p_index, unsigned long base_timestamp);
+
+public:
+	typedef std::string(*split_callback)(uint8_t bank_msb, uint8_t bank_lsb, uint8_t instrument);
+
+	void split_by_instrument_changes(split_callback cb = NULL);
+
     unsigned long get_subsong_count() const;
     unsigned long get_subsong( unsigned long p_index ) const;
 
@@ -247,7 +260,7 @@ public:
 
     void get_meta_data( unsigned long subsong, midi_meta_data & p_out );
 
-	void scan_for_loops( bool p_xmi_loops, bool p_marker_loops, bool p_rpgmaker_loops );
+    void scan_for_loops( bool p_xmi_loops, bool p_marker_loops, bool p_rpgmaker_loops, bool p_touhou_loops );
 
     static void encode_delta( std::vector<uint8_t> & p_out, unsigned long delta );
 };

Frameworks/midi_processing/midi_processing/midi_processor_hmp.cpp

@@ -49,6 +49,8 @@ bool midi_processor::process_hmp( std::vector<uint8_t> const& p_file, midi_conta
         if ( p_file.size() <= 0x4D )
             return false;
         dtx = ( p_file[ 0x4C ] << 16 ) | p_file[ 0x4D ];
+        if ( !dtx ) // dtx == 0, will cause division by zero on tempo calculations
+            return false;
     }
 
     p_out.initialize( 1, dtx );

Frameworks/midi_processing/midi_processing/midi_processor_mids.cpp

@@ -30,6 +30,8 @@ bool midi_processor::process_mids( std::vector<uint8_t> const& p_file, midi_cont
         time_format = it[ 0 ] | ( it[ 1 ] << 8 ) | ( it[ 2 ] << 16 ) | ( it[ 3 ] << 24 );
         it += 4;
         fmt_size -= 4;
+        if ( !time_format ) // dtx == 0, will cause division by zero on tempo calculations
+            return false;
     }
     if ( fmt_size >= 4 )
     {

Frameworks/midi_processing/midi_processing/midi_processor_riff_midi.cpp

@@ -2,15 +2,42 @@
 
 #include <string.h>
 
+static inline bool it_equal( std::vector<uint8_t>::const_iterator it1, const char *it2, size_t length )
+{
+    for ( size_t i = 0; i < length; i++ )
+    {
+        if ( it1[ i ] != it2[ i ] )
+            return false;
+    }
+    return true;
+}
+
+static inline uint32_t toInt32LE( const uint8_t *in )
+{
+    return static_cast<uint32_t>( in[ 0 ] ) |
+           static_cast<uint32_t>( in[ 1 ] << 8 ) |
+           static_cast<uint32_t>( in[ 2 ] << 16 ) |
+           static_cast<uint32_t>( in[ 3 ] << 24 );
+}
+
+static inline uint32_t toInt32LE( std::vector<uint8_t>::const_iterator in )
+{
+    return static_cast<uint32_t>( in[ 0 ] ) |
+           static_cast<uint32_t>( in[ 1 ] << 8 ) |
+           static_cast<uint32_t>( in[ 2 ] << 16 ) |
+           static_cast<uint32_t>( in[ 3 ] << 24 );
+}
+
+
 bool midi_processor::is_riff_midi( std::vector<uint8_t> const& p_file )
 {
     if ( p_file.size() < 20 ) return false;
-    if ( p_file[ 0 ] != 'R' || p_file[ 1 ] != 'I' || p_file[ 2 ] != 'F' || p_file[ 3 ] != 'F' ) return false;
+    if ( memcmp( &p_file[ 0 ], "RIFF", 4 ) != 0 ) return false;
     uint32_t riff_size = p_file[ 4 ] | ( p_file[ 5 ] << 8 ) | ( p_file[ 6 ] << 16 ) | ( p_file[ 7 ] << 24 );
     if ( riff_size < 12 || ( p_file.size() < riff_size + 8 ) ) return false;
-    if ( p_file[ 8 ] != 'R' || p_file[ 9 ] != 'M' || p_file[ 10 ] != 'I' || p_file[ 11 ] != 'D' ||
-         p_file[ 12 ] != 'd' || p_file[ 13 ] != 'a' || p_file[ 14 ] != 't' || p_file[ 15 ] != 'a' ) return false;
-    uint32_t data_size = p_file[ 16 ] | ( p_file[ 17 ] << 8 ) | ( p_file[ 18 ] << 16 ) | ( p_file[ 19 ] << 24 );
+    if ( memcmp( &p_file[ 8 ], "RMID", 4 ) != 0 ||
+         memcmp( &p_file[ 12 ], "data", 4 ) != 0 ) return false;
+    uint32_t data_size = toInt32LE(&p_file[ 16 ]);
     if ( data_size < 18 || p_file.size() < data_size + 20 || riff_size < data_size + 12 ) return false;
     std::vector<uint8_t> test;
     test.assign( p_file.begin() + 20, p_file.begin() + 20 + 18 );
@@ -34,7 +61,7 @@ bool midi_processor::process_riff_midi_count( std::vector<uint8_t> const& p_file
         if ( body_end - it < 8 ) return false;
         uint32_t chunk_size = it[ 4 ] | ( it[ 5 ] << 8 ) | ( it[ 6 ] << 16 ) | ( it[ 7 ] << 24 );
         if ( (unsigned long)(body_end - it) < chunk_size ) return false;
-        if ( it[ 0 ] == 'd' && it[ 1 ] == 'a' && it[ 2 ] == 't' && it[ 3 ] == 'a' )
+        if ( it_equal(it, "data", 4) )
         {
             std::vector<uint8_t> midi_file;
             midi_file.assign( it + 8, it + 8 + chunk_size );
@@ -92,9 +119,9 @@ bool midi_processor::process_riff_midi( std::vector<uint8_t> const& p_file, midi
     while ( it != body_end )
     {
         if ( body_end - it < 8 ) return false;
-        uint32_t chunk_size = it[ 4 ] | ( it[ 5 ] << 8 ) | ( it[ 6 ] << 16 ) | ( it[ 7 ] << 24 );
+        uint32_t chunk_size = toInt32LE( it + 4 );
         if ( (unsigned long)(body_end - it) < chunk_size ) return false;
-        if ( it[ 0 ] == 'd' && it[ 1 ] == 'a' && it[ 2 ] == 't' && it[ 3 ] == 'a' )
+        if ( it_equal( it, "data", 4 ) )
         {
             if ( !found_data )
             {
@@ -107,22 +134,23 @@ bool midi_processor::process_riff_midi( std::vector<uint8_t> const& p_file, midi
             it += 8 + chunk_size;
             if ( chunk_size & 1 && it != body_end ) ++it;
         }
-        else if ( it[ 0 ] == 'D' && it[ 1 ] == 'I' && it[ 2 ] == 'S' && it[ 3 ] == 'P' )
+        else if ( it_equal( it, "DISP", 4 ) )
         {
-            uint32_t type = it[ 8 ] | ( it[ 9 ] << 8 ) | ( it[ 10 ] << 16 ) | ( it[ 11 ] << 24 );
+            uint32_t type = toInt32LE( it + 8 );
             if ( type == 1 )
             {
-                extra_buffer.resize( chunk_size - 4 );
+                extra_buffer.resize( chunk_size - 4 + 1 );
                 std::copy( it + 12, it + 8 + chunk_size, extra_buffer.begin() );
+				extra_buffer[ chunk_size - 4 ] = '\0';
                 meta_data.add_item( midi_meta_data_item( 0, "display_name", (const char *) &extra_buffer[0] ) );
             }
             it += 8 + chunk_size;
             if ( chunk_size & 1 && it != body_end ) ++it;
         }
-        else if ( it[ 0 ] == 'L' && it[ 1 ] == 'I' && it[ 2 ] == 'S' && it[ 3 ] == 'T' )
+        else if ( it_equal( it, "LIST", 4 ) )
         {
             std::vector<uint8_t>::const_iterator chunk_end = it + 8 + chunk_size;
-            if ( it[ 8 ] == 'I' && it[ 9 ] == 'N' && it[ 10 ] == 'F' && it[ 11 ] == 'O' )
+            if ( it_equal( it + 8, "INFO", 4 ) )
             {
                 if ( !found_info )
                 {
@@ -131,7 +159,7 @@ bool midi_processor::process_riff_midi( std::vector<uint8_t> const& p_file, midi
                     while ( it != chunk_end )
                     {
                         if ( chunk_end - it < 4 ) return false;
-                        uint32_t field_size = it[ 4 ] | ( it[ 5 ] << 8 ) | ( it[ 6 ] << 16 ) | ( it[ 7 ] << 24 );
+                        uint32_t field_size = toInt32LE( it + 4 );
                         if ( (unsigned long)(chunk_end - it) < 8 + field_size ) return false;
                         std::string field;
                         field.assign( it, it + 4 );
@@ -143,8 +171,9 @@ bool midi_processor::process_riff_midi( std::vector<uint8_t> const& p_file, midi
                                 break;
                             }
                         }
-                        extra_buffer.resize( field_size );
+                        extra_buffer.resize( field_size + 1 );
                         std::copy( it + 8, it + 8 + field_size, extra_buffer.begin() );
+						extra_buffer[ field_size ] = '\0';
                         it += 8 + field_size;
                         meta_data.add_item( midi_meta_data_item( 0, field.c_str(), ( const char * ) &extra_buffer[0] ) );
                         if ( field_size & 1 && it != chunk_end ) ++it;

Frameworks/midi_processing/midi_processing/midi_processor_standard_midi.cpp

@@ -5,6 +5,8 @@ bool midi_processor::is_standard_midi( std::vector<uint8_t> const& p_file )
     if ( p_file.size() < 18 ) return false;
     if ( p_file[ 0 ] != 'M' || p_file[ 1 ] != 'T' || p_file[ 2 ] != 'h' || p_file[ 3 ] != 'd') return false;
     if ( p_file[ 4 ] != 0 || p_file[ 5 ] != 0 || p_file[ 6 ] != 0 || p_file[ 7 ] != 6 ) return false;
+    if ( p_file[ 10 ] == 0 && p_file[ 11 ] == 0 ) return false; // no tracks
+    if ( p_file[ 12 ] == 0 && p_file[ 13 ] == 0 ) return false; // dtx == 0, will cause division by zero on tempo calculations
     if ( p_file[ 14 ] != 'M' || p_file[ 15 ] != 'T' || p_file[ 16 ] != 'r' || p_file[ 17 ] != 'k' ) return false;
     return true;
 }
@@ -67,7 +69,7 @@ bool midi_processor::process_standard_midi_track( std::vector<uint8_t>::const_it
         {
             if ( last_sysex_length )
             {
-                track.add_event( midi_event( last_sysex_timestamp, midi_event::extended, 0, &buffer[0], last_sysex_length + 1 ) );
+                track.add_event( midi_event( last_sysex_timestamp, midi_event::extended, 0, &buffer[0], last_sysex_length ) );
                 last_sysex_length = 0;
             }
 
@@ -95,7 +97,7 @@ bool midi_processor::process_standard_midi_track( std::vector<uint8_t>::const_it
         {
             if ( last_sysex_length )
             {
-                track.add_event( midi_event( last_sysex_timestamp, midi_event::extended, 0, &buffer[0], last_sysex_length + 1 ) );
+                track.add_event( midi_event( last_sysex_timestamp, midi_event::extended, 0, &buffer[0], last_sysex_length ) );
                 last_sysex_length = 0;
             }
 
@@ -115,7 +117,7 @@ bool midi_processor::process_standard_midi_track( std::vector<uint8_t>::const_it
             int data_count = decode_delta( it, end );
             if ( data_count < 0 ) return false;
             if ( end - it < data_count ) return false;
-            buffer.resize( last_sysex_length + data_count + 1 );
+            buffer.resize( last_sysex_length + data_count );
             std::copy( it, it + data_count, buffer.begin() + last_sysex_length );
             it += data_count;
             last_sysex_length += data_count;
@@ -124,7 +126,7 @@ bool midi_processor::process_standard_midi_track( std::vector<uint8_t>::const_it
         {
             if ( last_sysex_length )
             {
-                track.add_event( midi_event( last_sysex_timestamp, midi_event::extended, 0, &buffer[0], last_sysex_length + 1 ) );
+                track.add_event( midi_event( last_sysex_timestamp, midi_event::extended, 0, &buffer[0], last_sysex_length ) );
                 last_sysex_length = 0;
             }
 
@@ -181,6 +183,9 @@ bool midi_processor::process_standard_midi( std::vector<uint8_t> const& p_file,
     uint16_t track_count_16 = ( it[2] << 8 ) | it[3];
     uint16_t dtx = ( it[4] << 8 ) | it[5];
 
+    if ( !track_count_16 || !dtx )
+        return false;
+
     it += 6;
 
     std::size_t track_count = track_count_16;

Plugins/MIDI/MIDI/MIDIDecoder.mm

@@ -62,7 +62,7 @@ static OSType getOSType(const char * in_)
     
 	track_num = [[[s url] fragment] intValue]; //What if theres no fragment? Assuming we get 0.
     
-    midi_file.scan_for_loops( true, true, true );
+    midi_file.scan_for_loops( true, true, true, true );
 
     framesLength = midi_file.get_timestamp_end( track_num, true );