# GNU Solfege - free ear training software
# Copyright (C) 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008 Tom Cato Amundsen
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.


import errno
import textwrap
import glob
import xrandom
import uuid
import stat
import time
import locale
import filesystem
import utils

import re, os, sys
import random
import cfg
import mpd
from mpd import mpdutils
from mpd.rat import Rat
import soundcard
import dataparser
from dataparser import istr
import osutils

_test_mode = False

class LessonfileException(Exception):
    pass

class MusicObjectException(mpd.MpdException):
    pass

class LessonfileParseException(Exception):
    pass

class NoQuestionsInFileException(LessonfileParseException):
    """
    Raised by find_random_question if the lesson file contains
    no questions at all.
    """
    def __init__(self, lessonfilename):
        LessonfileParseException.__init__(self, _("The lesson file contains no questions"))

class FileNotFound(LessonfileException):
    def __init__(self, filename):
        LessonfileException.__init__(self, _("The external file '%s' was not found") % filename)

class NoQuestionsConfiguredException(LessonfileException):
    """
    This exception is raised by select_random_question if the user has
    unselected all the questions available in the lesson file.
    """
    def __init__(self):
        LessonfileException.__init__(self,
            _("No questions selected"),
            _("You can select questions on the config page of the exercise."))


class plstring(unicode):
    pass

# The keys in the dict say how many steps up or down in the
# circle of fifths we go if we transpose.
_keys_to_interval = {
              -10: '-M6', #c -> eses
               -9: '-a2', #c -> beses
               -8: '-a5', #c -> fes
               -7: '-au', #c -> ces
               -6: '-a4', #c -> ges
               -5: 'm2', # c -> des
               -4: '-M3',# c -> as
               -3: 'm3', # c -> es
               -2: '-M2', # c -> bes
               -1: 'p4', # c -> f
                0: 'p1',
                1: '-p4', # c -> g,
                2: 'M2', # c -> d
                3: '-m3',# c -> a
                4: 'M3', # c -> e
                5: '-m2', # c -> b
                6: 'a4', # c -> fis
                7: 'au', #c -> cis
                8: 'a5', # c -> gis
                9: 'a2', # c -> dis
                10: 'M6', # c -> ais
            }

keywords = (
    # exercise modules
    'dictation',
    'rhythm',
    'harmonicprogressiondictation',
    'singchord',
    'singanswer',
    'chordvoicing',
    'chord',
    'compareintervals',
    'idbyname',
    'singinterval',
    'melodicinterval',
    'harmonicinterval',
    'example',
    'idtone',
    'twelvetone',
    'identifybpm',
    'nameinterval',
    'elembuilder',
    'rhythmtapping',
    'rhythmtapping2',
    'idproperty',
    #
    'harmonic',
    'melodic',
    'progression',
    'normal',
    'voice',
    'rvoice',
    'satb',
    'rhythm',
    'cmdline',
    'wavfile',
    'midifile',
    'horiz',
    'vertic',
    'accidentals',
    'key',
    'semitones',
    'play',
    'show',
    # elembuilder:
    'auto',
    # Used by the rhythm module:
    'newline',
    'd1', 'p1', 'a1',
    'd2', 'm2', 'M2', 'a2',
    'd3', 'm3', 'M3', 'a3',
    'd4', 'p4', 'a4',
    'd5', 'p5', 'a5',
    'd6', 'm6', 'M6', 'a6',
    'd7', 'm7', 'M7', 'a7',
    'd8', 'p8', 'a8',
    'd9', 'm9', 'M9', 'a9',
    'd10', 'm10', 'M10', 'a10',
)
predef = {
    'tempo': (60, 4),
    'yes': True,
    'no': False,
}
for n in keywords:
    predef[n] = n

lessonfile_functions = {
        '_': dataparser.dataparser_i18n_func,
        '_i': dataparser.dataparser_i18n__i_func,
        # play_wav should probably be removed. Replaced by wavfile
        'play_wav': lambda f: Wavfile(f),
        'music': lambda m: Music(m),
        'chord': lambda m: Chord(m),
        'satb': lambda m: Satb(m),
        'voice': lambda m: Voice(m),
        'rvoice': lambda m: Rvoice(m),
        'rhythm': lambda m: Rhythm(m),
        'percussion': lambda m: Percussion(m),
        'cmdline': lambda m: Cmdline(m),
        'wavfile': lambda m: Wavfile(m),
        'midifile': lambda m: Midifile(m),
        'mp3file': lambda m: Mp3file(m),
        'oggfile': lambda m: Oggfile(m),
        'progressionlabel': lambda s: plstring(s),
}

class _Header(dict):
    def __init__(self, headerdict):
        dict.__init__(self, headerdict)
        for key, value in (
                           ('version', ''),
                           ('title', ''),
                           ('description', ''),
                           ('musicformat', 'normal'),
                           ('random_transpose', True),
                           ('labelformat', 'normal'),
                           ('fillnum', 1),
                           ('filldir', 'horiz'),
                           ('have_repeat_slowly_button', False),
                           ('have_repeat_arpeggio_button', False),
                           ('at_question_start', []),
                           ('have_music_displayer', False),
                           ('enable_right_click', True),
                           ('disable_unused_intervals', True),
                           ):
            if key not in self:
                self[key] = value
    def __getattr__(self, name):
        """
        This function let us write

          header.variable_name

        as a shortcut or:

            header['variable_name']
        """
        if name in self:
            return self[name]
        return istr("")


class MusicBaseClass(object):
    def __init__(self, musicdata):
        self.m_musicdata = musicdata
    def get_mpd_music_string(self, lessonfile_ref):
        return "%s:%s" % (self.__class__.__name__, self.m_musicdata)


class MpdParsable(MusicBaseClass):
    """
    MpdParsable implements two generic play and play_slowly methods
    that can play the music if the subclass implements get_mpd_music_string.
    Music classes with more special needs will overwrite these play methods.
    """
    def play(self, lessonfile_ref, question):
        assert isinstance(lessonfile_ref, LessonfileCommon)
        instrument = lessonfile_ref.prepare_instrument_list(question)
        mpd.play_music(self.get_mpd_music_string(lessonfile_ref),
            lessonfile_ref.get_tempo(), instrument[0], instrument[1])
        if _test_mode:
            return self.get_mpd_music_string(lessonfile_ref)
    def play_slowly(self, lessonfile_ref, question):
        assert isinstance(lessonfile_ref, LessonfileCommon)
        instrument = lessonfile_ref.prepare_instrument_list(question)
        tempo = lessonfile_ref.get_tempo()
        tempo = (tempo[0]/2, tempo[1])
        mpd.play_music(self.get_mpd_music_string(lessonfile_ref),
            tempo, instrument[0], instrument[1])
        if _test_mode:
            return self.get_mpd_music_string(lessonfile_ref)
    def get_err_context(self, exception, lessonfile_ref):
        """
        Return a twoline string showing what caused the exception.
        """
        # I had to comment out the assert statement below, becuase the rhythmtapping
        # exercise will analyze the m_musicdata string directly and not the data from
        # get_mpd_music_string. And then the error can be on line 0.
        #assert exception.m_lineno != 0, "This nevers happend"
        first = exception.m_linepos1
        last = exception.m_linepos2
        if self.m_musicdata.count("\n") + 1 < exception.m_lineno:
            return "(FIXME: we need better error reporting)\nBad input to the music object of type %s causes the\nfollowing generated music code:" % self.__class__.__name__.lower() + "\n" +  self.get_mpd_music_string(lessonfile_ref)
        return "\n".join((self.m_musicdata.split("\n")[exception.m_lineno-1],
            " " * first + "^" * (last - first)))


class MpdDisplayable(MpdParsable):
    pass

class MpdTransposable(MpdDisplayable):
    pass

class ChordCommon(MpdTransposable):
    pass

class Chord(ChordCommon):
    def __init__(self, musicdata):
        ChordCommon.__init__(self, musicdata)
    def get_lilypond_code(self, lessonfile_ref):
        return r"\score{ "\
               r" \transpose c' %s{ <%s> }"\
               r" \layout { "\
               r"  ragged-last = ##t "\
               r'  \context { \Staff \remove "Time_signature_engraver" } '\
               r" }"\
               r"}"  % (
            lessonfile_ref.m_transpose.get_octave_notename(),
            self.m_musicdata)
    def get_lilypond_code_first_note(self, lessonfile_ref):
        return r"\transpose c' %s{ %s }" % (
            lessonfile_ref.m_transpose.get_octave_notename(),
            self.m_musicdata.split()[0])
    def get_mpd_music_string(self, lessonfile_ref):
        assert isinstance(lessonfile_ref, LessonfileCommon)
        if lessonfile_ref.header.random_transpose:
            return "\\staff\\transpose %s{<\n%s\n>}" \
                % (lessonfile_ref.m_transpose.get_octave_notename(),
                   self.m_musicdata)
        return "\\staff{<\n%s\n>}" % self.m_musicdata
    def get_music_as_notename_list(self, lessonfile_ref):
        """
        This method will validate the notenames, and raise a
        mpd.musicalpitch.InvalidNotenameException with the
        m_linepos1, m_linepos2 og m_lineno.
        """
        assert isinstance(lessonfile_ref, LessonfileCommon)
        try:
            if not lessonfile_ref.header.random_transpose:
                return [mpd.MusicalPitch.new_from_notename(n).get_octave_notename() for n in self.m_musicdata.split()]
            else:
                return [mpd.MusicalPitch.new_from_notename(n).transpose_by_musicalpitch(lessonfile_ref.m_transpose).get_octave_notename() for n in self.m_musicdata.split()]
        except mpd.InvalidNotenameException, e:
            e.m_lineno, e.m_linepos1, e.m_linepos2 = mpdutils.validate_only_notenames(self.m_musicdata)
            raise
    def get_music_as_notename_string(self, lessonfile_ref):
        return " ".join(self.get_music_as_notename_list(lessonfile_ref))
    def play(self, lessonfile_ref, question):
        self.__play(lessonfile_ref, question, question.tempo)
    def play_slowly(self, lessonfile_ref, question):
        self.__play(lessonfile_ref, question,
                (question.tempo[0] /2, question.tempo[1]))
    def __play(self, lessonfile_ref, question, tempo):
        assert isinstance(lessonfile_ref, LessonfileCommon)
        instrument = lessonfile_ref.prepare_instrument_list(question)
        assert len(instrument) in (2, 6)
        if len(instrument) == 2:
            mpd.play_music(self.get_mpd_music_string(lessonfile_ref),
                tempo, instrument[0], instrument[1])
        else:
            assert len(instrument) == 6
            t1 = soundcard.Track()
            t2 = soundcard.Track()
            t3 = soundcard.Track()
            t1.set_bpm(tempo[0])
            nlist = self.get_music_as_notename_list(lessonfile_ref)
            t1.set_patch(instrument[0])
            t2.set_patch(instrument[2])
            t3.set_patch(instrument[4])
            # start notes
            t1.note(4, mpd.notename_to_int(nlist[0]), instrument[1])
            for notename in nlist[1:-1]:
                t2.start_note(mpd.notename_to_int(notename), instrument[3])
            t2.notelen_time(4)
            for notename in nlist[1:-1]:
                t2.stop_note(mpd.notename_to_int(notename), instrument[3])
            t3.note(4, mpd.notename_to_int(nlist[-1]), instrument[5])
            soundcard.synth.play_track(t1, t2, t3)
    def play_arpeggio(self, lessonfile_ref, question):
        assert isinstance(lessonfile_ref, LessonfileCommon)
        # We have a problem here because the music need to know
        # things from the question it belongs to.
        instrument = lessonfile_ref.prepare_instrument_list(question)
        assert len(instrument) in (2, 6)
        if len(instrument) == 2:
            m = self.get_music_as_notename_string(lessonfile_ref)
            mpd.play_music(r"\staff{%s}" % m, cfg.get_int('config/arpeggio_bpm'),
                       instrument[0], instrument[1])
        else:
            assert len(instrument) == 6
            t1 = soundcard.Track()
            t2 = soundcard.Track()
            t3 = soundcard.Track()
            t1.set_bpm(cfg.get_int('config/arpeggio_bpm'))
            nlist = self.get_music_as_notename_list(lessonfile_ref)
            # set patches
            t1.set_patch(instrument[0])
            t2.set_patch(instrument[2])
            t3.set_patch(instrument[4])
            # start notes
            t1.note(4, mpd.notename_to_int(nlist[0]), instrument[1])
            t2.notelen_time(4)
            t3.notelen_time(4)
            for notename in nlist[1:-1]:
                t2.note(4, mpd.notename_to_int(notename), instrument[3])
                t3.notelen_time(4)
            t3.note(4, mpd.notename_to_int(nlist[-1]), instrument[5])
            soundcard.synth.play_track(t1, t2, t3)


class VoiceCommon(MpdTransposable):
    def get_first_pitch(self):
        a, b = mpdutils.find_possible_first_note(self.m_musicdata)
        return self.m_musicdata[a:b]

class Voice(VoiceCommon):
    def __init__(self, musicdata):
        VoiceCommon.__init__(self, musicdata)
    def get_lilypond_code(self, lessonfile_ref):
        return r"\transpose c' %s{ %s }" % (
            lessonfile_ref.m_transpose.get_octave_notename(),
            self.m_musicdata)
    def get_lilypond_code_first_note(self, lessonfile_ref):
        return r"\score{" \
               r" \new Staff<< "\
               r"  \new Voice\transpose c' %s{ \cadenzaOn %s }"\
               r"  \new Voice{ \hideNotes %s } "\
               r" >>"\
               r" \layout { ragged-last = ##t } " \
               r"}" % (
            lessonfile_ref.m_transpose.get_octave_notename(),
            self.get_first_pitch(),
            self.m_musicdata,
            )
    def get_mpd_music_string(self, lessonfile_ref):
        assert isinstance(lessonfile_ref, LessonfileCommon)
        if lessonfile_ref.header.random_transpose:
            return "\\staff\\transpose %s{\n%s\n}" \
                % (lessonfile_ref.m_transpose.get_octave_notename(),
                   self.m_musicdata)
        return "\\staff{\n%s\n}" % self.m_musicdata

class Rvoice(VoiceCommon):
    def __init__(self, musicdata):
        VoiceCommon.__init__(self, musicdata)
    def get_err_context(self, exception, lessonfile_ref):
        """
        Return a twoline string showing what caused the exception.
        """
        if exception.m_lineno == 0:
            # If there is an error in the first pitch, this will cause the
            # mpd parser to raise and exception on the \relative XXX line,
            # and this is on line 0.
            a, b = mpdutils.find_possible_first_note(self.m_musicdata)
            line = utils.string_get_line_at(self.m_musicdata, a)
            a, b = mpdutils.find_possible_first_note(line)
            return "\n".join((line, " " * (a) + "^" * (b - a)))
        else:
            return VoiceCommon.get_err_context(self, exception, lessonfile_ref)
    def get_mpd_music_string(self, lessonfile_ref):
        assert isinstance(lessonfile_ref, LessonfileCommon)
        a, b = mpdutils.find_possible_first_note(self.m_musicdata)
        if (a, b) != (None, None):
            first_pitch = self.m_musicdata[a:b]
        else:
            first_pitch = None
        if lessonfile_ref.header.random_transpose:
            transpose_str = r"\transpose %s" % lessonfile_ref.m_transpose.get_octave_notename()
        else:
            transpose_str = ""
        if not first_pitch:
            # Returning there is mostly theoretical. Why would anyone
            # have a music object with no notes, only commands like
            # \clef, \staff etc?
            return "\\staff%s{\n%s\n}"\
               % (transpose_str, self.m_musicdata)
        new_musicdata = self.m_musicdata[:a] + first_pitch.rstrip(",'") + self.m_musicdata[b:]
        return "\\staff%s\\relative %s{\n%s\n}" \
          % (transpose_str, first_pitch, new_musicdata)
    def get_lilypond_code(self, lessonfile_ref):
        return r"\transpose c' %s\relative c{ %s }" % (
            lessonfile_ref.m_transpose.get_octave_notename(),
            self.m_musicdata)
    def get_lilypond_code_first_note(self, lessonfile_ref):
        return r"\score{" \
               r" \new Staff<< "\
               r"  \new Voice\transpose c' %s\relative c{ \cadenzaOn %s }" \
               r"  \new Voice{ \hideNotes %s } "\
               r" >> "\
               r" \layout { ragged-last = ##t } " \
               r"}" % (
            lessonfile_ref.m_transpose.get_octave_notename(),
            self.get_first_pitch(),
            self.m_musicdata,
            )

class Satb(ChordCommon):
    def __init__(self, musicdata):
        ChordCommon.__init__(self, musicdata)
        if "\n" in self.m_musicdata:
            self._m_orig_musicdata = self.m_musicdata
            self.m_musicdata = self.m_musicdata.replace("\n", "")
    def get_mpd_music_string(self, lessonfile_ref):
        assert isinstance(lessonfile_ref, LessonfileCommon)
        v = [n.strip() for n in self.m_musicdata.split('|')]
        if len(v) != 4:
            raise MusicObjectException("Satb music should be divided into 4 parts by the '|' character")
        if [x for x in self.m_musicdata.split("|") if not x.strip()]:
            raise MusicObjectException("Satb music does not allow an empty voice")
        #FIXME BUG BUG BUG this only works for the currently active question
        if 'key' in lessonfile_ref.get_question():
            k = lessonfile_ref.get_question()['key']
        else:
            k = "c \major"
        music = "\\staff{ \key %s\\stemUp <%s> }\n" \
                "\\addvoice{ \\stemDown <%s> }\n" \
                "\\staff{ \key %s\\clef bass \\stemUp <%s>}\n"\
                "\\addvoice{ \\stemDown <%s>}" % (k, v[0], v[1], k, v[2], v[3])
        if lessonfile_ref.header.random_transpose:
            music = music.replace(r"\staff",
                      r"\staff\transpose %s" % lessonfile_ref.m_transpose.get_octave_notename())
            music = music.replace(r"\addvoice",
                      r"\addvoice\transpose %s" % lessonfile_ref.m_transpose.get_octave_notename())
        return music
    def play_arpeggio(self, lessonfile_ref, question):
        assert isinstance(lessonfile_ref, LessonfileCommon)
        instrument = lessonfile_ref.prepare_instrument_list(question)
        track = soundcard.Track()
        track.set_bpm(cfg.get_int('config/default_bpm'))
        track.set_patch(instrument[0])
        voices = [n.strip() for n in self.m_musicdata.split('|')]
        for x in 0, 1:
            s = voices[x].strip().split(" ")
            for n in s:
                if lessonfile_ref.header.random_transpose:
                    n = mpd.MusicalPitch.new_from_notename(n).transpose_by_musicalpitch(lessonfile_ref.m_transpose).get_octave_notename()
                if cfg.get_string('user/sex') == 'female':
                    track.note(4, mpd.notename_to_int(n), instrument[1])
                else:
                    track.note(4, mpd.notename_to_int(n)-12, instrument[1])
        for x in 2, 3:
            s = voices[x].strip().split(" ")
            for n in s:
                if lessonfile_ref.header.random_transpose:
                    n = mpd.MusicalPitch.new_from_notename(n).transpose_by_musicalpitch(lessonfile_ref.m_transpose).get_octave_notename()
                if cfg.get_string('user/sex') == 'male':
                    track.note(4, mpd.notename_to_int(n), instrument[1])
                else:
                    track.note(4, mpd.notename_to_int(n)+12, instrument[1])
        soundcard.synth.play_track(track)
    def get_err_context(self, exception, lessonfile_ref):
        """
        Return a twoline string showing what caused the exception.
        """
        if len(self.m_musicdata.split("|")) != 4:
            return self.m_musicdata
        if [x for x in self.m_musicdata.split("|") if not x.strip()]:
            return self.m_musicdata
        line1 = []
        line2 = []
        err_found = False
        try:
            if self._m_orig_musicdata:
                bad_err_msg = "\n".join(textwrap.wrap("The music code from the lesson file has been modified by removing the new-line characters. This to more easily show where the error occured. Satb music should not contain music characters.", 60)) + "\n"
        except AttributeError:
            bad_err_msg = ""
        for i, s in enumerate(self.m_musicdata.split("|")):
            line1.append(s)
            if i == exception.m_lineno:
                line2.append("^" * len(s))
                err_found = True
            elif not err_found:
                line2.append(" " * len(s))
        return bad_err_msg + ("\n".join(("|".join(line1), " ".join(line2))))


class PercBaseClass(MpdParsable):
    def get_mpd_music_string(self, lessonfile_ref):
        return "\\staff{\n%s\n}" % self.m_musicdata
    def _gen_track(self, lessonfile_ref, question):
        score = mpd.parser.parse_to_score_object(self.get_mpd_music_string(lessonfile_ref))
        track = score.get_midi_events_as_percussion(cfg.get_int('config/preferred_instrument_velocity'))[0]
        track.prepend_bpm(lessonfile_ref.get_tempo()[0],
                          lessonfile_ref.get_tempo()[1])
        return track


class Rhythm(PercBaseClass):
    def __init__(self, musicdata):
        PercBaseClass.__init__(self, musicdata)
    def play(self, lessonfile_ref, question):
        track = self._gen_track(lessonfile_ref, question)
        track.replace_note(mpd.notename_to_int("c"), 37)
        track.replace_note(mpd.notename_to_int("d"), 80)
        soundcard.synth.play_track(track)


class Percussion(PercBaseClass):
    def __init__(self, musicdata):
        PercBaseClass.__init__(self, musicdata)
    def play(self, lessonfile_ref, question):
        soundcard.synth.play_track(self._gen_track(lessonfile_ref, question))

class _MusicExternalPlayer(MusicBaseClass):
    def __init__(self, typeid, musicdata):
        MusicBaseClass.__init__(self, musicdata)
        self.m_typeid = typeid
    def play(self, lessonfile_ref, question):
        assert isinstance(lessonfile_ref, LessonfileCommon)
        musicfile = os.path.join(lessonfile_ref.m_location, self.m_musicdata)
        if os.path.exists(musicfile):
            soundcard.play_mediafile(self.m_typeid, musicfile)
        else:
            raise FileNotFound(musicfile)
    def get_err_context(self, exception, lessonfile_ref):
        return ""

class Midifile(_MusicExternalPlayer):
    def __init__(self, musicdata):
        _MusicExternalPlayer.__init__(self, 'midi', musicdata)


class Wavfile(_MusicExternalPlayer):
    def __init__(self, musicdata):
        _MusicExternalPlayer.__init__(self, 'wav', musicdata)

class Mp3file(_MusicExternalPlayer):
    def __init__(self, musicdata):
        _MusicExternalPlayer.__init__(self, 'mp3', musicdata)

class Oggfile(_MusicExternalPlayer):
    def __init__(self, musicdata):
        _MusicExternalPlayer.__init__(self, 'ogg', musicdata)

class Cmdline(MusicBaseClass):
    def __init__(self, musicdata):
        MusicBaseClass.__init__(self, musicdata)
    def play(self, lessonfile_ref, question):
        assert isinstance(lessonfile_ref, LessonfileCommon)
        osutils.run_external_program(str(self.m_musicdata),
                                     lessonfile_ref.m_location, "")


class Music(MpdDisplayable):
    def __init__(self, musicdata):
        MpdDisplayable.__init__(self, musicdata)
    def get_mpd_music_string(self, lessonfile_ref):
        if lessonfile_ref.header.random_transpose:
            s = self.m_musicdata.replace(r'\staff',
               r'\staff\transpose %s' % lessonfile_ref.m_transpose.get_octave_notename())
            s = s.replace(r'\addvoice',
               r'\addvoice\transpose %s' % lessonfile_ref.m_transpose.get_octave_notename())
            return s
        return self.m_musicdata
    def play(self, lessonfile_ref, question):
        assert isinstance(lessonfile_ref, LessonfileCommon)
        if len(lessonfile_ref.prepare_instrument_list(question)) == 2:
            MpdParsable.play(self, lessonfile_ref, question)
        else:
            self.play_3patches(lessonfile_ref, question, 
                    lessonfile_ref.get_tempo())
    def play_slowly(self, lessonfile_ref, question):
        assert isinstance(lessonfile_ref, LessonfileCommon)
        if len(lessonfile_ref.prepare_instrument_list(question)) == 2:
            MpdParsable.play_slowly(self, lessonfile_ref, question)
        else:
            tempo = lessonfile_ref.get_tempo()
            tempo = (tempo[0]/2, tempo[1])
            self.play_3patches(lessonfile_ref, question, tempo)
    def play_3patches(self, lessonfile_ref, question, tempo):
        """
        Play the music with different instrument for the top and bottom voice.
        Will use the instruments defined in the preferences window.
        """
        mpd.play_music3(
            self.get_mpd_music_string(lessonfile_ref),
            tempo,
            lessonfile_ref.prepare_instrument_list(question))
    def get_err_context(self, exception, lessonfile_ref):
        """
        Return a twoline string showing what caused the exception.
        """
        first = exception.m_linepos1
        last = exception.m_linepos2
        if lessonfile_ref.header.random_transpose:
            s = self.m_musicdata.replace(r'\staff',
               r'\staff\transpose %s' % lessonfile_ref.m_transpose.get_octave_notename())
            s = s.replace(r'\addvoice',
               r'\addvoice\transpose %s' % lessonfile_ref.m_transpose.get_octave_notename())
            i = len(s) - len(self.m_musicdata)
        else:
            i = 0
        first = exception.m_linepos1 - i
        last = exception.m_linepos2 - i
        return "\n".join((self.m_musicdata.split("\n")[exception.m_lineno],
            " " * first + "^" * (last - first)))

def parse_test_def(s):
    m = re.match("(\d+)\s*x", s)
    count = int(m.groups()[0])
    return (count, 'x')


class LessonfileCommon(object):
    def __init__(self):
        self.m_prev_question = None
        # This variable stores the directory the lesson file is located in.
        # We need this to we can find other files relative to this file.
        # .parse_file will set it to the location of the file.
        self.m_location = "."
        self._idx = None
        self.m_filename = "<STRING>"
    def parse_file(self, filename):
        """Parse the file named filename. Set these variables:
        self.header     a Header instance
        self.questions  a list of all question
        """
        self.m_location = os.path.split(filename)[0]
        self.m_filename = filename
        self.parse_string(open(filename, 'rU').read(), really_filename=filename)
    def parse_string(self, s, really_filename=None):
        """
        See parse_file docstring.
        """
        self.dataparser = dataparser.Dataparser(predef, lessonfile_functions, ('tempo',))
        self.dataparser.m_location = self.m_location
        try:
            self.dataparser.parse_string(s, really_filename)
        except LessonfileParseException, e:
            e.m_nonwrapped_text = self.dataparser._lexer.get_err_context(self.dataparser._lexer.pos - 2)
            e.m_token = self.dataparser._lexer.m_tokens[self.dataparser._lexer.pos - 2]
            raise
        self.m_transpose = mpd.MusicalPitch.new_from_notename("c'")
        self.header = _Header(self.dataparser.header)
        self.m_globals = self.dataparser.globals
        self.m_questions = self.dataparser.questions
        self.blocklists = self.dataparser.blocklists
        del self.dataparser
        for question in self.m_questions:
            question.active = 1
            # FIXMECOMPAT
            if 'music' in question and isinstance(question.music, basestring):
                # The following line is for backward compatibility
                question.music = Music(question.music)
        self.m_random = xrandom.Random(range(len(self.m_questions)))
        if self.header.random_transpose == True:
            self.header.random_transpose = ['key', -5, 5]
        # Backward compatability to handle old style
        # random_transpose = -4, 5 FIXMECOMPAT
        if self.header.random_transpose and len(self.header.random_transpose) == 2:
            self.header.random_transpose \
                = ['semitones'] + self.header.random_transpose
        # Some variables does only make sense if we have a music displayer
        if self.header.at_question_start:
            self.header.have_music_displayer = True


class QuestionsLessonfile(LessonfileCommon):
    def __init__(self):
        LessonfileCommon.__init__(self)
        self.m_discards = []
    def select_random_question(self):
        """
        Select a new question by random. It will use the music in the
        lesson file question variable 'music' when selecting transposition.
        """
        # when we start the program with --no-random, we want to go
        # throug all the questions in the lesson file in sequential order.
        if cfg.get_bool('config/no_random'):
            try:
                self.m_no_random_idx
            except:
                self.m_no_random_idx = 0
            self.header.random_transpose = False

        count = 0
        available_question_idx = []
        for i in range(len(self.m_questions)):
            if self.m_questions[i]['active']:
                available_question_idx.append(i)
        if not available_question_idx:
            raise NoQuestionsConfiguredException()
        while 1:
            count += 1
            if cfg.get_bool('config/no_random'):
                if self.m_no_random_idx < len(available_question_idx):
                    self._idx = self.m_no_random_idx
                    self.m_no_random_idx += 1
                else:
                    self._idx = self.m_no_random_idx = 0
            else:
                if cfg.get_string("app/random_function") == 'random_by_random':
                    self._idx = self.m_random.random_by_random(available_question_idx)
                elif cfg.get_string("app/random_function") == 'random_by_random2':
                    self._idx = self.m_random.random_by_random2(available_question_idx)
                elif cfg.get_string("app/random_function") == 'random_by_selection':
                    self._idx = self.m_random.random_by_selection(available_question_idx)
                else:
                    self._idx = random.choice(available_question_idx)
            if self.header.random_transpose:
                self.m_transpose = self.find_random_transpose()
            if count == 10:
                break
            if self.m_prev_question == self.get_music() \
                and (len(self.m_questions) > 1 or self.header.random_transpose):
                continue
            break
        self.m_random.add(self._idx)
        self.m_prev_question = self.get_music()
    def find_random_transpose(self):
        """
        Return a MusicalPitch representing a suggested random
        transposition for the currently selected question,
        m_questions[self._idx]
        """
        if 'key' in self.m_questions[self._idx]:
            key = self.m_questions[self._idx]['key']
        else:
            key = "c \major"
        if self.header.random_transpose == True:
            self.header.random_transpose = ['key', -5, 5]
        if self.header.random_transpose[0] == 'semitones':
            retval = self.semitone_find_random_transpose()
            if random.randint(0, 1):
                retval.enharmonic_flip()
        else:
            retval = self._xxx_find_random_transpose(key)
        return retval
    def semitone_find_random_transpose(self):
        """
        Called to find random transposition in "semitone" mode.
        Create and return a random MusicalPitch representing this transposition.
        """
        assert self.header.random_transpose[0] == 'semitones'
        return mpd.MusicalPitch().randomize(
              mpd.transpose_notename("c'", self.header.random_transpose[1]),
              mpd.transpose_notename("c'", self.header.random_transpose[2]))
    def _xxx_find_random_transpose(self, key):
        """
        Called to create random transposition in "accidentals" or "key" mode.
        Create and return a random MusicalPitch representing this transposition.
        Keyword arguments:
        key -- the key the question is written in, for example "c \major"
        """
        assert self.header.random_transpose[0] in ('key', 'accidentals')
        low, high = self.header.random_transpose[1:3]
        tone, minmaj = key.split()
        k = mpd.MusicalPitch.new_from_notename(tone).get_octave_notename()
        #FIXME this list say what key signatures are allowed in sing-chord
        # lesson files. Get the correct values and document them.
        kv = ['des', 'aes', 'ees', 'bes', 'f', 'c',
              'g', 'd', 'a', 'e', 'b', 'fis', 'cis', 'gis']
        # na tell the number of accidentals (# is positive, b is negative)
        # the question has from the lessonfile before anything is transpose.
        na = kv.index(k) - 5
        if minmaj == '\\minor':
            na -= 3
        if self.header.random_transpose[0] == 'accidentals':
            # the number of steps down the circle of fifths we can go
            n_down = low - na
            # the number of steps up the circle of fifths we can go
            n_up = high - na
        else:
            assert self.header.random_transpose[0] == 'key'
            n_down = low
            n_up = high
        interv = mpd.Interval()
        interv.set_from_string(_keys_to_interval[random.choice(range(n_down, n_up+1))])
        return mpd.MusicalPitch.new_from_notename("c'") + interv
    def iterate_questions_with_unique_names(self):
        """Iterate the questions in the lessonfile, but only yield the
        first question if several questions have the same name. The
        untranslated name is used when deciding if a name is unique.
        """
        names = {}
        for question in self.m_questions:
            if 'name' in question and question.name.cval not in names:
                names[question.name.cval] = 1
                yield question
    def get_unique_cnames(self):
        """Return a list of all cnames in the file, in the same order
        as they appear in the file. Only list each cname once, even if
        there are more questions with the same cname.
        """
        names = []
        for question in self.m_questions:
            if 'name' in question and question.name.cval not in names:
                names.append(question.name.cval)
        return names
    def get_question(self):
        """
        Return the currently selected question.
        """
        assert self._idx is not None
        return self.m_questions[self._idx]
    def get_tempo(self):
        assert self._idx is not None
        return self.m_questions[self._idx].tempo
    def get_name(self):
        """
        Return the translated name of the currently selected question.
        """
        assert self._idx is not None
        return self.m_questions[self._idx].name
    def get_cname(self):
        """
        The 'cname' of a question is the C locale of the question name.
        Said easier: If the lesson file supplies translations, then 'cname'
        is the untranslated name.
        """
        assert self._idx is not None
        return self.m_questions[self._idx].name.cval
    def get_lilypond_code(self):
        assert self._idx is not None
        return self.m_questions[self._idx].music.get_lilypond_code(self)
    def get_lilypond_code_first_note(self):
        assert self._idx is not None
        return self.m_questions[self._idx].music.get_lilypond_code_first_note(self)
    def get_music(self, varname='music'):
        """
        Return the music for the currently selected question. This is complete
        music code that can be fed to mpd.play_music(...).

        If the music type not of a type that mpd.play_music can handle,
        for example a midi file or a cmdline type, then we return a string
        that can be used to compare if the music of two questions are equal.
        This string is not parsable by any functions and should only be used
        to compare questions.
        """
        assert self._idx is not None
        return self.m_questions[self._idx][varname].get_mpd_music_string(self)
    def get_music_as_notename_list(self, varname):
        """
        Return a list of notenames from the variabale VARNAME in the
        currently selected question. The notes are transposed if
        header.random_transpose is set.
        """
        assert self._idx is not None
        return self.get_question()[varname].get_music_as_notename_list(self)
    def get_music_as_notename_string(self, varname):
        """
        Return a string with notenames representing the question currently
        selected question. The notes are transposed if
        header.random_transpose is set.
        """
        return " ".join(self.get_music_as_notename_list(varname))
    def has_question(self):
        """
        Return True if a question is selected.
        """
        return self._idx is not None
    def parse_string(self, s, really_filename=None):
        super(QuestionsLessonfile, self).parse_string(s, really_filename)
        if not self.m_questions:
            raise NoQuestionsInFileException(self.m_filename)
    def play_question(self, question=None, varname='music'):
        """Play the question. Play the current question if question is none.
        varname is the name of the variable that contains the music.
        """
        if not question:
            question = self.get_question()
        try:
            question[varname].play(self, question)
        except mpd.MpdException, e:
            # This code have to be here for code that run m_P.play_question
            # exception_handled to be able to say which variable has the bug
            # and show the bad code.
            if 'm_mpd_varname' not in dir(e):
                e.m_mpd_varname = varname
            if 'm_mpd_badcode' not in dir(e):
                e.m_mpd_badcode = question[varname].get_err_context(e, self)
            raise
    def play_question_slowly(self, question=None, varname='music'):
        if not question:
            question = self.get_question()
        question[varname].play_slowly(self, question)
    def play_question_arpeggio(self, varname='music'):
        self.get_question()[varname].play_arpeggio(self, self.get_question())
    def prepare_instrument_list(self, question):
        """Return a list created from the instrument variable the question.
        Use app default values if the variable is missing.
        Convert instrument names to integer values.
        Returns: lowest, middle, highest
        """
        if cfg.get_bool('config/override_default_instrument'):
            instrument = [cfg.get_int('config/lowest_instrument'),
                         cfg.get_int('config/lowest_instrument_velocity'),
                         cfg.get_int('config/middle_instrument'),
                         cfg.get_int('config/middle_instrument_velocity'),
                         cfg.get_int('config/highest_instrument'),
                         cfg.get_int('config/highest_instrument_velocity')]
        elif 'instrument' in question:
            instrument = question['instrument']
        elif 'instrument' in self.m_globals:
            instrument = self.m_globals['instrument']
        else:
            instrument = [cfg.get_int('config/preferred_instrument'),
                          cfg.get_int('config/preferred_instrument_velocity')]
        if isinstance(instrument, (unicode, int)):
            instrument = [instrument,
                          cfg.get_int('config/preferred_instrument_velocity')]
        assert len(instrument) in (2, 6)
        if len(instrument) == 2:
            if isinstance(instrument[0], unicode):
                try:
                    instrument[0] = soundcard.find_midi_instrument_number(instrument[0])
                except KeyError, e:
                    print >> sys.stderr, "Warning: Invalid instrument name '%s' in lesson file:" % instrument[0], e
                    instrument[0] = cfg.get_int('config/preferred_instrument')
            if not (0 <= instrument[1] < 128):
                print >> sys.stderr, "Warning: Adjusting instrument velocity since this value is invalid '%s'" % instrument[1]
                instrument[1] = cfg.get_int('config/preferred_instrument_velocity')
        elif len(instrument) == 6:
            for x in (0, 2, 4):
                if isinstance(instrument[x], unicode):
                    try:
                        instrument[x] = soundcard.find_midi_instrument_number(instrument[x])
                    except KeyError, e:
                        print "Warning: Invalid instrument name in lesson file:", e
                        instrument[0] = cfg.get_int('config/preferred_instrument')
                if not (0 <= instrument[x+1] < 128):
                    instrument[x+1] = cfg.get_int('config/preferred_instrument_velocity')
        return instrument
    def discard_questions_without_name(self):
        # Delete questions that does not have a name
        q = self.m_questions
        self.m_questions = []
        for idx, question in enumerate(q):
            if 'name' not in question:
                self.m_discards.append("\n".join(textwrap.wrap(_('Discarding question %(questionidx)i from the lessonfile "%(filename)s" because it is missing the "name" variable. All questions in lesson files of this type must have a name variable.' % {'questionidx': idx, 'filename': self.m_filename}))))
                continue
            else:
                self.m_questions.append(question)

class TestSupport(object):
    """
    Lessonfile classes can add this class to the list of classes it
    inherits from if the exercise want to have tests.
    """
    def _generate_test_questions(self):
        count, t = parse_test_def(self.header.test)
        q = range(len(self.m_questions)) * count
        random.shuffle(q)
        return q
    def get_test_requirement(self):
        """
        Return the amount of exercises that has to be correct to
        pass the test. (values 0.0 to 1.0)
        """
        m = re.match("([\d\.]+)%", self.header.test_requirement)
        return float(m.groups()[0])/100.0
    def enter_test_mode(self):
        self.m_test_questions = self._generate_test_questions()
        self.m_test_idx = -1
    def next_test_question(self):
        assert self.m_test_idx < len(self.m_test_questions)
        self.m_test_idx += 1
        self._idx = self.m_test_questions[self.m_test_idx]
        if self.header.random_transpose:
            old = self.m_transpose
            # try really hard not to get the same tonika:
            for x in range(100):
                self.m_transpose = self.find_random_transpose()
                if old != self.m_transpose:
                    break
    def is_test_complete(self):
        """
        Return True if the test is compleded.
        """
        return self.m_test_idx == len(self.m_test_questions) -1

class HeaderLessonfile(LessonfileCommon):
    """
    This lesson file class should be used by all the exercise modules
    that does not need any question blocks defined.
    """
    pass

class DictationLessonfile(QuestionsLessonfile):
    def get_breakpoints(self):
        assert self._idx is not None
        r = []
        if 'breakpoints' in self.m_questions[self._idx]:
            r = self.m_questions[self._idx]['breakpoints']
            if not type(r) == type([]):
                r = [r]
        r = map(lambda e: Rat(e[0], e[1]), r)
        return r
    def get_clue_end(self):
        assert self._idx is not None
        if 'clue_end' in self.m_questions[self._idx]:
            try:
                return Rat(*self.m_questions[self._idx]['clue_end'])
            except TypeError:
                raise LessonfileException("The 'clue_end' variable was not well formed")
    def get_clue_music(self):
        assert self._idx is not None
        if 'clue_music' in self.m_questions[self._idx]:
            return self.m_questions[self._idx]['clue_music']
    def select_previous(self):
        """
        Select the previous question. Do nothing if we are on the first
        question.
        """
        assert self._idx is not None
        if self._idx > 0:
            self._idx = self._idx - 1
    def select_next(self):
        """
        Select the next question. Do nothing if we are on the last question.
        """
        assert self._idx is not None
        if self._idx < len(self.m_questions) -1:
            self._idx = self._idx + 1
    def select_first(self):
        """
        Select the first question.
        """
        self._idx = 0

class SingChordLessonfile(QuestionsLessonfile):
    pass

class NameIntervalLessonfile(HeaderLessonfile):
    def parse_string(self, s, really_filename=None):
        super(NameIntervalLessonfile, self).parse_string(s, really_filename)
        iquality = []
        inumbers = []
        self.header.intervals = [mpd.Interval(n) for n in self.header.intervals]
        for i in self.header.intervals:
            if i.get_quality_short() not in iquality:
                iquality.append(i.get_quality_short())
            if i.steps() not in inumbers:
                inumbers.append(i.steps())
        def quality_sort(a, b):
            v = ['dd', 'd', 'm', 'M', 'p', 'a', 'aa']
            return cmp(v.index(a), v.index(b))
        iquality.sort(quality_sort)
        inumbers.sort()
        if not self.header.interval_number:
            self.header.interval_number = inumbers
        if not isinstance(self.header.interval_number, list):
            self.header.interval_number = [self.header.interval_number]
        if not self.header.interval_quality:
            self.header.interval_quality = iquality
        if not isinstance(self.header.interval_quality, list):
            self.header.interval_number = [self.header.interval_quality]
        if self.header.accidentals == "":
            self.header.accidentals = 1
        if self.header.clef == "":
            self.header.clef = u"violin"
        if not self.header.tones:
            self.header.tones = [mpd.MusicalPitch.new_from_notename("b"),
                                mpd.MusicalPitch.new_from_notename("g''")]
        else:
            if len(self.header.tones) != 2:
                raise LessonfileParseException("The length of the lesson file header variable 'tones' has to be 2")
            self.header.tones = [mpd.MusicalPitch.new_from_notename(n) for n in self.header.tones]

class IdByNameLessonfile(QuestionsLessonfile, TestSupport):
    def __init__(self):
        QuestionsLessonfile.__init__(self)
        TestSupport.__init__(self)
    def parse_string(self, s, really_filename=None):
        super(IdByNameLessonfile, self).parse_string(s, really_filename)
        # Also, if some questions has cuemusic, then we need the displayer
        if [q for q in self.m_questions if 'cuemusic' in q]:
            self.header.have_music_displayer = True
        self.discard_questions_without_name()

class SingAnswerLessonfile(QuestionsLessonfile):
    def parse_string(self, s, really_filename=None):
        super(SingAnswerLessonfile, self).parse_string(s, really_filename)
        v = [q for q in self.m_questions if 'question_text' not in q]
        if [q for q in self.m_questions if 'question_text' not in q]:
            raise LessonfileParseException(_('Question number %(index)i in the lesson file "%(filename)s" is missing the "question_text" variable.') % {
                'index': self.m_questions.index(v[0]),
                'filename': self.m_filename})

class IntervalsLessonfile(HeaderLessonfile, TestSupport):
    """
    Common lesson file class for some interval exercises.
    We inherit from TestSupport, but overwrites some methods from it.
    """
    def enter_test_mode(self):
        count, t = parse_test_def(self.header.test)
        if self.header.intervals:
            self.m_test_questions = self.header.intervals * count
        else:
            self.m_test_questions = self.header.ask_for_intervals_0 * count
        random.shuffle(self.m_test_questions)
        self.m_test_idx = -1
    def next_test_question(self):
        self.m_test_idx += 1

class IdPropertyLessonfile(QuestionsLessonfile):
    def parse_string(self, s, really_filename=None):
        """
        Call IdPropertyLessonfile.parse_string and set the self.m_props dict.
        Change some question variables, so that:

          inversion = 0

        is the same as

          inversion = _("root position")
        """
        super(IdPropertyLessonfile, self).parse_string(s, really_filename)
        if self.header.flavour == 'chord':
            if not self.header.new_button_label:
                self.header.new_button_label = _("_New chord")
            if not self.header.lesson_heading:
                self.header.lesson_heading = _("Identify the chord")
            if not self.header.qprops:
                self.header.qprops = ['name', 'inversion', 'toptone']
                self.header.qprop_labels = [
                             istr.new_translated("Chord type", _("Chord type")),
                             istr.new_translated("Inversion", _("Inversion")),
                             istr.new_translated("Toptone", _("Toptone"))]
        if not self.header.qprops:
            raise LessonfileParseException(_("Missing qprops variable in the lesson file %s.") % self.m_filename)
        # These two tests are needed, so we can have qprops and qprop_labels
        # lists with only one element.
        if not isinstance(self.header.qprops, list):
            self.header.qprops = [self.header.qprops]
        if not isinstance(self.header.qprop_labels, list):
            self.header.qprop_labels = [self.header.qprop_labels]
        if len(self.header.qprops) != len(self.header.qprop_labels):
            raise LessonfileParseException(_("Error in the lesson file header of \"%(filename)s\". The variables qprops and qprop_labels must have the same length.") % {'filename': self.m_filename})
        # m_props will be a dict where each key is the property var name.
        # The values will be a list of possible values for that property.
        # The values are of type istr. This mean that .cval holds the
        # C locale string.
        self.m_props = {}
        for k in self.header.qprops:
            self.m_props[k] = []
        for question in self.m_questions:
            for varname in self.header.qprops:
                if varname in question:
                    if varname == 'inversion':
                        if question[varname] == 0:
                            question[varname] = istr(_("root position"))
                            question[varname].cval= "root position"
                        elif type(question[varname]) == int \
                            and question[varname] > 0:
                            i = question[varname]
                            question[varname] = istr(_("%i. inversion") % i)
                            question[varname].cval = "%i. inversion" % i
                    # FIXMECOMPAT convert integer properties to strings.
                    # This to be compatible with solfege 3.9.1 and older.
                    if type(question[varname]) in (int, float):
                        question[varname] = istr(unicode(question[varname]))
                    # then add to m_props
                    if question[varname] not in self.m_props[varname]:
                        self.m_props[varname].append(question[varname])
        for k in self.m_props.keys():
            if not self.m_props[k]:
                idx = self.header.qprops.index(k)
                del self.header.qprops[idx]
                del self.header.qprop_labels[idx]
        for k in [k for k in self.m_props if not self.m_props[k]]:
            del self.m_props[k]
        # Have to use [:] when deleting from the list
        for idx, question in enumerate(self.m_questions[:]):
            # The list we create has the name of all the missing properties
            # in the question
            missing_props = [p for p in self.m_props if p not in question]
            if missing_props:
                self.m_discards.append("\n".join(textwrap.wrap(ungettext(
                    'Discarding question %(questionidx)i from the lesson file "%(filename)s" because of a missing variable: %(var)s',
                    'Discarding question %(questionidx)i from the lesson file "%(filename)s" because of some missing variables: %(var)s',
                    len(missing_props)) %  {'questionidx': idx, 'filename': self.m_filename, 'var': ", ".join(missing_props)})))
                self.m_questions[idx] = None
        self.m_questions = [q for q in self.m_questions if q is not None]

class ChordLessonfile(IdPropertyLessonfile):
    def parse_string(self, s, really_filename=None):
        super(ChordLessonfile, self).parse_string(
        s.replace("header {",
                """
                header {
                   qprops = "name", "inversion", "toptone"
                   qprop_labels = _("Chord type"), _("Inversion"), _("Chord type")
                """),
        really_filename)


class ElembuilderLessonfile(QuestionsLessonfile):
    def parse_string(self, s, really_filename=None):
        super(ElembuilderLessonfile, self).parse_string(s, really_filename)
        # We need the name variable for statistics
        self.discard_questions_without_name()


class LessonFileManager:
    def __init__(self, debug):
        # A list of info about lesson files that where discarded
        # by the lesson file manager
        self.m_discards = []
        self.parse(debug)
    def parse(self, debug):
        lessonpath = ['lesson-files',
                      filesystem.user_lessonfiles()]
        if debug:
            lessonpath.append('regression-lesson-files')
        self.m_uiddb = {}
        vim_tmpfile_re = re.compile("\..*\.swp")
        for dir in lessonpath:
            if not os.path.isdir(dir):
                print "warning: invalid directory in path:", dir
                continue
            v = glob.glob(os.path.join(dir, "*"))
            for filename in v:
                filename = filename.decode(locale.getpreferredencoding())
                # since I usually run solfege from the source dir:
                if os.path.split(filename)[-1] in ('.arch-ids', 'Makefile', 'Makefile.in') \
                  or not os.path.isfile(filename):
                    continue
                # We save the returned lesson_id because we need it below
                lesson_id = self.parse_into_uiddb(filename)
                if not lesson_id:
                    continue
                # We have to check if the lesson file has changed,
                # and if it has, then the results has to be deleted.
                # This because Solfege has no way of knowing what kind
                # of changes has been done to the lesson file.
                h = hash(open(filename, 'r').read())
                hash_filename = os.path.join(filesystem.app_data(),
                           "testresults", "%s_hash" % lesson_id)
                if os.path.exists(hash_filename):
                    h2 = int(open(hash_filename, 'r').read())
                    if h != h2:
                        dirname = os.path.join(filesystem.app_data(),
                                "testresults", lesson_id)
                        for f in os.listdir(dirname):
                            os.remove(os.path.join(dirname, f))
    def parse_into_uiddb(self, filename):
                """
                Returns the lesson_id of the parsed file.
                Append to .m_discards and return None if there was any errors.
                """
                try:
                    f = LessonIdParser(filename)
                    if not f.has_lesson_id():
                        f.add_lesson_id()
                except IOError, e:
                    self.m_discards.append({
                        'filename': filename,
                        'exception': e,
                    })
                    return
                msg = 'Failed to parse the lessonfile "%s". Do not report this as a bug. Fix the file, or delete it.'
                if os.path.isabs(filename):
                    msg = msg % filename
                else:
                    msg = msg % (os.path.join(os.getcwdu(), filename))
                msg = "\n".join(textwrap.wrap(msg, 110)) + "\n"
                try:
                    p = parse_lesson_file_header(filename)
                except dataparser.DataparserException, e:
                    print >> sys.stderr, msg
                    print >> sys.stderr, str(e)
                    return
                if not p:
                    print >> sys.stderr, msg
                    return
                if not 'title' in p.header:
                    p.header['title'] = '@@@NOT TITLE'
                if 'module' not in p.header:
                    self.m_discards.append({
                        'filename': filename,
                        'reason': 'no module',
                    })
                    return
                n = {
                   'filename': filename,
                   'mtime': os.stat(filename)[stat.ST_MTIME],
                   'header': p.header,
                }
                if not p.header['lesson_id'] in self.m_uiddb:
                    self.m_uiddb[p.header['lesson_id']] = n
                else:
                    if type(self.m_uiddb[p.header['lesson_id']]) != list:
                        self.m_uiddb[p.header['lesson_id']] = \
                            [self.m_uiddb[p.header['lesson_id']]]
                    self.m_uiddb[p.header['lesson_id']].append(n)
                return p.header['lesson_id']
    def create_lessonfile_index(self):
        self.m_htmldoc = """<html><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"></head><body><p>%s</p>""" % _("This page lists all the lesson files that Solfege can find. You can click on the links to start practising.")
        d = {}
        for lesson_id, n in self.m_uiddb.items():
            if not 'module' in n['header']:
                print "warning: %s is missing a module declaration" %  self.get(lesson_id, 'filename')
                continue
            if n['header']['module'] not in d:
                d[n['header']['module']] = []
            d[n['header']['module']].append(lesson_id)
        for module in d.keys():
            self.m_htmldoc = "%s<h2>%s</h2>" % (self.m_htmldoc, module)
            self.m_htmldoc += "<ul>"
            for n in d[module]:
                self.m_htmldoc += "<li><a href='solfege:practise/%s'>%s</a>: %s</li>" % (
                    n,
                    self.get(n, 'filename'),
                    self.get(n, 'title'))
            self.m_htmldoc += "</ul>"
    def get(self, lesson_id, fieldname):
        if fieldname in ('title', 'test', 'module', 'lesson_heading'):
            if fieldname in self.m_uiddb[lesson_id]['header']:
                return self.m_uiddb[lesson_id]['header'][fieldname]
        if fieldname in self.m_uiddb[lesson_id]:
            return self.m_uiddb[lesson_id][fieldname]
    def is_test_passed(self, lesson_id):
        return os.path.exists(os.path.join(filesystem.app_data(),
            'testresults', lesson_id, 'passed'))
    def ignore_duplicates_with_lesson_id(self, lesson_id):
        """
        Delete the duplicates with lesson_id, keep the oldest file.
        """
        def sort_func(a, b):
            return cmp(a['mtime'], b['mtime'])
        self.m_uiddb[lesson_id].sort(sort_func)
        self.m_uiddb[lesson_id] = self.m_uiddb[lesson_id][0]
    def delete_not_fn(self, lesson_id, fn):
        """
        Assumes the lesson_id has duplicate entries.
        Delete the entries that is not the file fn.
        """
        self.m_uiddb[lesson_id] = [d for d in self.m_uiddb[lesson_id] if d['filename'] == fn][0]
    def iterate_duplicated_lesson_id(self):
        """
        Return the a lesson_id if there exist lesson_ids that
        are duplicated. If not, return None.
        """
        for v in self.m_uiddb.values():
            if isinstance(v, list):
                yield v[0]['header']['lesson_id']
    def iterate_lesson_ids(self):
        for k in self.m_uiddb:
            yield k
    def get_lesson_file_info(self, lesson_id):
        """
        FIXME: the function name is not very good.
        Return data used to fix things when we have a lesson_id crash.
        """
        return [{'filename': d['filename'], 'timestr': time.strftime('%c', time.localtime(d['mtime'])), 'mtime': d['mtime']} for d in self.m_uiddb[lesson_id]]

def parse_lesson_file_header(filename):
    """
    This function is used at program starup to get the info the
    lessonfile_manager needs. This might not be bullet proof, but
    it provides a 22x speedup, and that was necessary when we got
    many lesson files.

    Return None if we find no header block.
    """
    r = re.compile("\\header\s*{.*?}", re.MULTILINE|re.DOTALL)
    s = open(filename, 'rU').read()
    m = r.search(s)
    p = dataparser.Dataparser(predef, lessonfile_functions)
    p.m_ignore_lookup_error = True
    if not m:
        return
    p.parse_string(m.group())
    return p


class LessonIdParser(object):
    """
    This is a light weight parser for lesson files that is only used when
    checking/adding/updating lesson_ids.
    """
    def __init__(self, filename):
        self.m_filename = filename
        f = open(filename, 'rU')
        self.m_file_content = f.read()
        f.close()
    def has_header_block(self):
        """
        Return True if the file has a header block.
        """
        return re.search("^\s*header", self.m_file_content, re.MULTILINE) is not None
    def has_lesson_id(self):
        """
        Return True if the lesson file has a lesson_id.
        """
        return re.search("lesson_id", self.m_file_content) is not None
    def new_lesson_id(self):
        """
        Generate and add a new lesson_id for the file filename.
        """
        if self.has_lesson_id():
            ofile = open(self.m_filename, 'w')
            m = re.search("lesson_id\s*=\s*\".+?\"", self.m_file_content)
            ofile.write(self.m_file_content[:m.start()])
            self.m_new_id = uuid.generate()
            ofile.write("lesson_id = \"%s\"" % self.m_new_id)
            ofile.write(self.m_file_content[m.end():])
            ofile.close()
        else:
            print "warning: new_lesson_id when we have no id. Ok anyway."
            self.add_lesson_id()
    def add_header_block(self):
        """
        Add a header block with a lesson id.
        """
        assert not self.has_header_block()
        v = self.m_file_content.split("\n")
        for i in range(len(v)):
            if not v[i].startswith('#'):
                break
        self.m_new_id = uuid.generate()
        v.insert(i+1, "header { \n    lesson_id=\"%s\"\n}" % self.m_new_id)
        self.m_file_content = "\n".join(v)
        ofile = open(self.m_filename, 'w')
        ofile.write(self.m_file_content)
        ofile.close()
    def add_lesson_id(self):
        """
        Add a lesson id to m_file_content and write it to disk.
        """
        assert not self.has_lesson_id()
        if not self.has_header_block():
            self.add_header_block()
        else:
            ofile = open(self.m_filename, 'w')
            m = re.search("^\s*header\s*{", self.m_file_content,
                            re.MULTILINE)
            if m:
                ofile.write(self.m_file_content[:m.start()])
                ofile.write("\nheader {")
                self.m_new_id = uuid.generate()
                ofile.write("\n    lesson_id = \"%s\"" % self.m_new_id)
                ofile.write(self.m_file_content[m.end():])
            ofile.close()