I had the same problem with my Event Receiver. I tried ListTemplateId, ListTemplateOwner, and even ListUrl. I knew the settings were valid, but they were being ignored and the receiver was being attached to every list.
I had a hunch that it might be related to the Event Receiver being declared within a Site scoped feature. This seems to be confirmed by the "documentation" for SPEventElement:
  switch (this.FeatureDefinition.Scope)
  {
    case SPFeatureScope.Site:
      if (this.SiteScopedReceivers())
      {
        this.UpdateEventReceiversForSite(site, sqlcmdAppendOnly, new SPEventElement.GetSqlToUpdateEventReceiversForSite(this.GetEventReceivers(site.RootWeb).GetSqlToAddEventReceiversToSite));
        break;
      }
      else
      {
        this.UpdateEventReceiversForWeb(site.RootWeb, sqlcmdAppendOnly, new SPEventElement.GetSqlToUpdateEventReceiversForWeb(this.GetEventReceivers(site.RootWeb).GetSqlToAddEventReceiversToWeb));
        break;
      }
    case SPFeatureScope.Web:
      if (this.RootWebOnly && !web.IsRootWeb)
      {
        ULS.SendTraceTag(1718513714U, (ULSCatBase) ULSCat.msoulscat_WSS_General, ULSTraceLevel.Verbose, "Event Receivers in Feature '{0}' were not activated because current web is not the root web.", new object[1]
        {
          (object) this.FeatureDefinition.Id.ToString("B")
        });
        break;
      }
      else
      {
        bool templateIdExists;
        int templateId;
        this.CheckTemplateId(out templateIdExists, out templateId);
        if (!templateIdExists)
        {
          if (this.ListUrl != null)
          {
            this.UpdateEventReceiversForList(web, sqlcmdAppendOnly, new SPEventElement.GetSqlToUpdateEventReceiversForList(this.GetEventReceivers(web).GetSqlToAddEventReceiversToList), true);
            break;
          }
          else
          {
            this.UpdateEventReceiversForWeb(web, sqlcmdAppendOnly, new SPEventElement.GetSqlToUpdateEventReceiversForWeb(this.GetEventReceivers(web).GetSqlToAddEventReceiversToWeb));
            break;
          }
        }
        else
        {
          if (this.ListUrl != null)
            throw new SPException(SPResource.GetString("ElementHasBothTemplateIdAndUrl", new object[0]));
          this.UpdateEventReceiversForListTemplate(templateId, web, sqlcmdAppendOnly, new SPEventElement.GetSqlToUpdateEventReceiversForList(this.GetEventReceivers(web).GetSqlToAddEventReceiversToList));
          break;
        }
      }
  }
It appears that ListTemplateId, ListTemplateOwner, and ListUrl are ignored for Site scoped features. When I moved my Event Receiver element to a Web scoped feature, the receiver was properly attached only to the desired list.