embedded: fix vtable specialization for nested classes

Fixes a crash in case of an inner class (with no generic parameters), which is nested inside another generic type, like
```
struct G<T> {
  class Inner {}
}
```

rdar://131311511

Author: eeckstein

lib/SILOptimizer/Transforms/VTableSpecializer.cpp

@@ -153,16 +153,19 @@ static bool specializeVTablesOfSuperclasses(SILModule &module,
 SILVTable *swift::specializeVTableForType(SILType classTy, SILModule &module,
                                 SILTransform *transform) {
   CanType astType = classTy.getASTType();
-  BoundGenericClassType *genClassTy = dyn_cast<BoundGenericClassType>(astType);
-  if (!genClassTy) return nullptr;
+  if (!astType->isSpecialized())
+    return nullptr;
+  NominalOrBoundGenericNominalType *genClassTy = dyn_cast<NominalOrBoundGenericNominalType>(astType);
+  ClassDecl *classDecl = astType->getClassOrBoundGenericClass();
+  if (!classDecl)
+    return nullptr;
 
   if (module.lookUpSpecializedVTable(classTy)) return nullptr;
 
   LLVM_DEBUG(llvm::errs() << "specializeVTableFor "
-                          << genClassTy->getDecl()->getName() << ' '
+                          << classDecl->getName() << ' '
                           << genClassTy->getString() << '\n');
 
-  ClassDecl *classDecl = genClassTy->getDecl();
   SILVTable *origVtable = module.lookUpVTable(classDecl);
   if (!origVtable) {
     // This cannot occur in regular builds - only if built without wmo, which
@@ -244,8 +247,8 @@ bool swift::specializeClassMethodInst(ClassMethodInst *cm) {
   SILValue instance = cm->getOperand();
   SILType classTy = instance->getType();
   CanType astType = classTy.getASTType();
-  BoundGenericClassType *genClassTy = dyn_cast<BoundGenericClassType>(astType);
-  if (!genClassTy) return false;
+  if (!astType->isSpecialized())
+    return false;
 
   SubstitutionMap subs = astType->getContextSubstitutionMap();
 

test/embedded/generic-classes.swift

@@ -34,6 +34,18 @@ class SubClass2 : SubClass1<Int> {
   override func test() { print("SubClass2") }
 }
 
+public class Outer<T> {
+  public class Inner {
+    func foo() {
+      print("Inner.foo")
+    }
+  }
+}
+
+public func makeInner() -> Outer<Int>.Inner {
+  return Outer<Int>.Inner()
+}
+
 @main
 struct Main {
   static func main() {
@@ -43,10 +55,12 @@ struct Main {
     makeItFoo(f: g)
     let x = SubClass2()
     x.test()
+    makeInner().foo()
   }
 }
 
 // CHECK: GenericFooableClass<T>.foo
 // CHECK: GenericFooableSubClass<T>.foo
 // CHECK: SubClass2
+// CHECK: Inner.foo