I would suggest to check the requirements separatedly. Example:
public static void main(String[] args) throws Exception {
    String [] fileNames = {"myfile.pdf","myfile.pdf.pdf","myfile.doc","myfile.txt","myfile.txt.doc"};         
    for(String fileName : fileNames){
        boolean valid = hasValidExtension(fileName) && noDuplicates(fileName);
        System.out.println(fileName+"\t" + (valid?"Valid":"Invalid"));
    }
}
public static boolean hasValidExtension(String fileName){
    String ext = fileName.split("\\.(?=[^\\.]+$)")[1];
    if(ext.equals("pdf")||ext.equals("doc")||ext.equals("docx")){
        return true;
    }
    return false;
}
public static boolean noDuplicates(String fileName){
    String[] splited = fileName.split("\\.");
    Set<String> set = new HashSet<>();
    for (String str : splited) {
        if (set.add(str) == false) {
           return false;
        }
    }
    return true;
}