You need to escape both { and } that are outside of a character class:
let rx = "typedef\\W+struct\\W+\\{([^}]*)\\}\\W+(\\w+);"
A quick demo:
let rx = "typedef\\W+struct\\W+\\{([^}]*)\\}\\W+(\\w+);"
let str = "typedef: struct { something } text;"
print(str.range(of: rx, options: .regularExpression) != nil) 
// => true
When the { and } are inside a character class they may stay unescaped (as in [^}]).
Using this code (answer by Confused Vorlon), you may get the first match with all capturing groups:
extension NSTextCheckingResult {
    func groups(testedString:String) -> [String] {
        var groups = [String]()
        for i in  0 ..< self.numberOfRanges
        {
            let group = String(testedString[Range(self.range(at: i), in: testedString)!])
            groups.append(group)
        }
        return groups
    }
}
let str = "typedef: struct { something } text;"
let rx = "typedef\\W+struct\\W+\\{([^}]*)\\}\\W+(\\w+);"
let MyRegex = try! NSRegularExpression(pattern: rx)
if let match = MyRegex.firstMatch(in: str, range: NSMakeRange(0, str.count)) {
     let groups = match.groups(testedString: str)
     print(groups)
}
// => ["typedef: struct { something } text;", " something ", "text"]